Add support for partials and path segments for Handlebars.

This introduces support for some missing features to the Handlebars lexer:
Partials and path segments. Partials mostly appeared to work before, but the
`>` in `{{> ... }}` would appear as a syntax error, as could other
components of the partial. This change introduces support for:

* Standard partials: `{{> partialName}}`
* Partials with parameters: `{{> partialName varname="value"}}`
* Ddynamic partials: `{{> (partialFunc)}}`
* Ddynamic partials with lookups: `{{> (lookup ../path "partialName")}}`
* Partial blocks: `{{> @partial-block}}`
* Inline partials: `{{#*inline}}..{{/inline}}`

It also introduces support for path segments, which can reference content in
the current context or in a parent context. For instance, `this.name`,
`this/name`, `./name`, `../name`, `this/name`, etc. These are all now tracked
as variables.

1 files changed, 158 insertions, 0 deletions

diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py
new file mode 100644
index 00000000..de8cd73b
--- /dev/null
+++ b/pygments/sphinxext.py
@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.sphinxext
+    ~~~~~~~~~~~~~~~~~~
+
+    Sphinx extension to generate automatic documentation of lexers,
+    formatters and filters.
+
+    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+import sys
+
+from docutils import nodes
+from docutils.statemachine import ViewList
+from sphinx.util.compat import Directive
+from sphinx.util.nodes import nested_parse_with_titles
+
+
+MODULEDOC = '''
+.. module:: %s
+
+%s
+%s
+'''
+
+LEXERDOC = '''
+.. class:: %s
+
+    :Short names: %s
+    :Filenames:   %s
+    :MIME types:  %s
+
+    %s
+
+'''
+
+FMTERDOC = '''
+.. class:: %s
+
+    :Short names: %s
+    :Filenames: %s
+
+    %s
+
+'''
+
+FILTERDOC = '''
+.. class:: %s
+
+    :Name: %s
+
+    %s
+
+'''
+
+
+class PygmentsDoc(Directive):
+    """
+    A directive to collect all lexers/formatters/filters and generate
+    autoclass directives for them.
+    """
+    has_content = False
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {}
+
+    def run(self):
+        self.filenames = set()
+        if self.arguments[0] == 'lexers':
+            out = self.document_lexers()
+        elif self.arguments[0] == 'formatters':
+            out = self.document_formatters()
+        elif self.arguments[0] == 'filters':
+            out = self.document_filters()
+        else:
+            raise Exception('invalid argument for "pygmentsdoc" directive')
+        node = nodes.compound()
+        vl = ViewList(out.split('\n'), source='')
+        nested_parse_with_titles(self.state, vl, node)
+        for fn in self.filenames:
+            self.state.document.settings.record_dependencies.add(fn)
+        return node.children
+
+    def document_lexers(self):
+        from pygments.lexers._mapping import LEXERS
+        out = []
+        modules = {}
+        moduledocstrings = {}
+        for classname, data in sorted(LEXERS.items(), key=lambda x: x[0]):
+            module = data[0]
+            mod = __import__(module, None, None, [classname])
+            self.filenames.add(mod.__file__)
+            cls = getattr(mod, classname)
+            if not cls.__doc__:
+                print("Warning: %s does not have a docstring." % classname)
+            docstring = cls.__doc__
+            if isinstance(docstring, bytes):
+                docstring = docstring.decode('utf8')
+            modules.setdefault(module, []).append((
+                classname,
+                ', '.join(data[2]) or 'None',
+                ', '.join(data[3]).replace('*', '\\*').replace('_', '\\') or 'None',
+                ', '.join(data[4]) or 'None',
+                docstring))
+            if module not in moduledocstrings:
+                moddoc = mod.__doc__
+                if isinstance(moddoc, bytes):
+                    moddoc = moddoc.decode('utf8')
+                moduledocstrings[module] = moddoc
+
+        for module, lexers in sorted(modules.items(), key=lambda x: x[0]):
+            if moduledocstrings[module] is None:
+                raise Exception("Missing docstring for %s" % (module,))
+            heading = moduledocstrings[module].splitlines()[4].strip().rstrip('.')
+            out.append(MODULEDOC % (module, heading, '-'*len(heading)))
+            for data in lexers:
+                out.append(LEXERDOC % data)
+
+        return ''.join(out)
+
+    def document_formatters(self):
+        from pygments.formatters import FORMATTERS
+
+        out = []
+        for classname, data in sorted(FORMATTERS.items(), key=lambda x: x[0]):
+            module = data[0]
+            mod = __import__(module, None, None, [classname])
+            self.filenames.add(mod.__file__)
+            cls = getattr(mod, classname)
+            docstring = cls.__doc__
+            if isinstance(docstring, bytes):
+                docstring = docstring.decode('utf8')
+            heading = cls.__name__
+            out.append(FMTERDOC % (heading, ', '.join(data[2]) or 'None',
+                                   ', '.join(data[3]).replace('*', '\\*') or 'None',
+                                   docstring))
+        return ''.join(out)
+
+    def document_filters(self):
+        from pygments.filters import FILTERS
+
+        out = []
+        for name, cls in FILTERS.items():
+            self.filenames.add(sys.modules[cls.__module__].__file__)
+            docstring = cls.__doc__
+            if isinstance(docstring, bytes):
+                docstring = docstring.decode('utf8')
+            out.append(FILTERDOC % (cls.__name__, name, docstring))
+        return ''.join(out)
+
+
+def setup(app):
+    app.add_directive('pygmentsdoc', PygmentsDoc)