From da4737ce98bd037e86d005cf35f3f08ec0f1cbe2 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 14 Mar 2014 18:03:19 -0400 Subject: - Template modules now generate a JSON "metadata" structure at the bottom of the source file which includes parseable information about the templates' source file, encoding etc. as well as a mapping of module source lines to template lines, thus replacing the "# SOURCE LINE" markers throughout the source code. The structure also indicates those lines that are explicitly not part of the template's source; the goal here is to allow integration with coverage tools. --- doc/build/changelog.rst | 11 +++++++++++ mako/codegen.py | 42 +++++++++++++++++++++++++++++++++++++----- mako/compat.py | 5 +++++ mako/exceptions.py | 26 ++++++++++++++++++-------- mako/pygen.py | 11 +++++++++-- setup.py | 13 ++++++++++--- 6 files changed, 90 insertions(+), 18 deletions(-) diff --git a/doc/build/changelog.rst b/doc/build/changelog.rst index 2a0e5f3..7664284 100644 --- a/doc/build/changelog.rst +++ b/doc/build/changelog.rst @@ -9,6 +9,17 @@ Changelog :version: 0.9.2 :released: + .. change:: + :tags: feature + + Template modules now generate a JSON "metadata" structure at the bottom + of the source file which includes parseable information about the + templates' source file, encoding etc. as well as a mapping of module + source lines to template lines, thus replacing the "# SOURCE LINE" + markers throughout the source code. The structure also indicates those + lines that are explicitly not part of the template's source; the goal + here is to allow integration with coverage tools. + .. change:: :tags: feature, py3k :pullreq: github:7 diff --git a/mako/codegen.py b/mako/codegen.py index 045d03c..ec587ba 100644 --- a/mako/codegen.py +++ b/mako/codegen.py @@ -14,7 +14,7 @@ from mako import util, ast, parsetree, filters, exceptions from mako import compat -MAGIC_NUMBER = 9 +MAGIC_NUMBER = 10 # names which are hardwired into the # template and are not accessed via the @@ -102,6 +102,8 @@ class _GenerateRenderMethod(object): self.last_source_line = -1 self.compiler = compiler self.node = node + self.source_map = {} + self.boilerplate_map = [] self.identifier_stack = [None] self.in_def = isinstance(node, (parsetree.DefTag, parsetree.BlockTag)) @@ -146,6 +148,27 @@ class _GenerateRenderMethod(object): for node in defs: _GenerateRenderMethod(printer, compiler, node) + if not self.in_def: + self.write_metadata_struct() + + def write_metadata_struct(self): + self.source_map[self.printer.lineno] = self.last_source_line + struct = { + "filename": self.compiler.filename, + "uri": self.compiler.uri, + "source_encoding": self.compiler.source_encoding, + "line_map": self.source_map, + "boilerplate_lines": self.boilerplate_map + } + self.mark_boilerplate() + self.printer.writelines( + '"""', + '__M_BEGIN_METADATA', + compat.json.dumps(struct), + '__M_END_METADATA\n' + '"""' + ) + @property def identifiers(self): return self.identifier_stack[-1] @@ -232,7 +255,7 @@ class _GenerateRenderMethod(object): [n.name for n in main_identifiers.topleveldefs.values()] ) - self.printer.write("\n\n") + self.printer.write_blanks(2) if len(module_code): self.write_module_code(module_code) @@ -251,6 +274,7 @@ class _GenerateRenderMethod(object): this could be the main render() method or that of a top-level def.""" + self.mark_boilerplate() if self.in_def: decorator = node.decorator if decorator: @@ -288,7 +312,7 @@ class _GenerateRenderMethod(object): self.write_def_finish(self.node, buffered, filtered, cached) self.printer.writeline(None) - self.printer.write("\n\n") + self.printer.write_blanks(2) if cached: self.write_cache_decorator( node, name, @@ -305,6 +329,7 @@ class _GenerateRenderMethod(object): def write_inherit(self, node): """write the module-level inheritance-determination callable.""" + self.mark_boilerplate() self.printer.writelines( "def _mako_inherit(template, context):", "_mako_generate_namespaces(context)", @@ -315,6 +340,7 @@ class _GenerateRenderMethod(object): def write_namespaces(self, namespaces): """write the module-level namespace-generating callable.""" + self.mark_boilerplate() self.printer.writelines( "def _mako_get_namespace(context, name):", "try:", @@ -401,7 +427,7 @@ class _GenerateRenderMethod(object): self.printer.writeline( "context.namespaces[(__name__, %s)] = ns" % repr(node.name)) - self.printer.write("\n") + self.printer.write_blanks(1) if not len(namespaces): self.printer.writeline("pass") self.printer.writeline(None) @@ -536,9 +562,12 @@ class _GenerateRenderMethod(object): """write a source comment containing the line number of the corresponding template line.""" if self.last_source_line != node.lineno: - self.printer.writeline("# SOURCE LINE %d" % node.lineno) + self.source_map[self.printer.lineno] = node.lineno self.last_source_line = node.lineno + def mark_boilerplate(self): + self.boilerplate_map.append(self.printer.lineno) + def write_def_decl(self, node, identifiers): """write a locally-available callable referencing a top-level def""" funcname = node.funcname @@ -606,6 +635,7 @@ class _GenerateRenderMethod(object): writes code to retrieve captured content, apply filters, send proper return value.""" + self.mark_boilerplate() if not buffered and not cached and not filtered: self.printer.writeline("return ''") if callstack: @@ -861,6 +891,7 @@ class _GenerateRenderMethod(object): pass def visitBlockTag(self, node): + self.mark_boilerplate() if node.is_anonymous: self.printer.writeline("%s()" % node.funcname) else: @@ -930,6 +961,7 @@ class _GenerateRenderMethod(object): n.accept_visitor(self) self.identifier_stack.pop() + self.mark_boilerplate() self.write_def_finish(node, buffered, False, False, callstack=False) self.printer.writelines( None, diff --git a/mako/compat.py b/mako/compat.py index c5ef84b..8e8ee70 100644 --- a/mako/compat.py +++ b/mako/compat.py @@ -94,6 +94,11 @@ except: return func(*(args + fargs), **newkeywords) return newfunc +if py26: + import json +else: + import simplejson as json + if not py25: def all(iterable): for i in iterable: diff --git a/mako/exceptions.py b/mako/exceptions.py index b8f97ee..523805f 100644 --- a/mako/exceptions.py +++ b/mako/exceptions.py @@ -167,14 +167,24 @@ class RichTraceback(object): None, None, None, None)) continue - template_ln = module_ln = 1 - line_map = {} - for line in module_source.split("\n"): - match = re.match(r'\s*# SOURCE LINE (\d+)', line) - if match: - template_ln = int(match.group(1)) - module_ln += 1 - line_map[module_ln] = template_ln + template_ln = 1 + + source_map = re.search( + r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", + module_source, re.S).group(1) + source_map = compat.json.loads(source_map) + line_map = dict( + (int(k), v) for k, v in source_map['line_map'].items() + ) + + for mod_line in reversed(sorted(line_map)): + tmpl_line = line_map[mod_line] + while mod_line > 0: + mod_line -= 1 + if mod_line in line_map: + break + line_map[mod_line] = tmpl_line + template_lines = [line for line in template_source.split("\n")] mods[filename] = (line_map, template_lines) diff --git a/mako/pygen.py b/mako/pygen.py index cba9464..d6559e5 100644 --- a/mako/pygen.py +++ b/mako/pygen.py @@ -26,6 +26,9 @@ class PythonPrinter(object): # the stream we are writing to self.stream = stream + # current line number + self.lineno = 0 + # a list of lines that represents a buffered "block" of code, # which can be later printed relative to an indent level self.line_buffer = [] @@ -34,8 +37,9 @@ class PythonPrinter(object): self._reset_multi_line_flags() - def write(self, text): - self.stream.write(text) + def write_blanks(self, num=1): + self.stream.write("\n" * num) + self.lineno += num def write_indented_block(self, block): """print a line or lines of python which already contain indentation. @@ -94,6 +98,7 @@ class PythonPrinter(object): # write the line self.stream.write(self._indent_line(line) + "\n") + self.lineno += 1 # see if this line should increase the indentation level. # note that a line can both decrase (before printing) and @@ -213,11 +218,13 @@ class PythonPrinter(object): for entry in self.line_buffer: if self._in_multi_line(entry): self.stream.write(entry + "\n") + self.lineno += 1 else: entry = entry.expandtabs() if stripspace is None and re.search(r"^[ \t]*[^# \t]", entry): stripspace = re.match(r"^([ \t]*)", entry).group(1) self.stream.write(self._indent_line(entry, stripspace) + "\n") + self.lineno += 1 self.line_buffer = [] self._reset_multi_line_flags() diff --git a/setup.py b/setup.py index 04d4551..9f1e8f2 100644 --- a/setup.py +++ b/setup.py @@ -13,10 +13,17 @@ markupsafe_installs = ( sys.version_info >= (2, 6) and sys.version_info < (3, 0) ) or sys.version_info >= (3, 3) +json_installs = ( + sys.version_info < (2, 6) + ) + +install_requires = [] + if markupsafe_installs: - install_requires = ['MarkupSafe>=0.9.2'] -else: - install_requires = [] + install_requires.append('MarkupSafe>=0.9.2') + +if json_installs: + install_requires.append('simplejson') setup(name='Mako', version=VERSION, -- cgit v1.2.1