New docs + website using Sphinx.

4 files changed, 158 insertions, 5 deletions

diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py
index 3376b309..68260cf8 100644
--- a/pygments/formatters/other.py
+++ b/pygments/formatters/other.py
@@ -40,7 +40,7 @@ class RawTokenFormatter(Formatter):
 
     The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
     be converted to a token stream with the `RawTokenLexer`, described in the
-    `lexer list <lexers.txt>`_.
+    :doc:`lexer list <lexers>`.
 
     Only two options are accepted:
 
diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py
index 8e9fad6a..f2b946e8 100644
--- a/pygments/lexers/other.py
+++ b/pygments/lexers/other.py
@@ -3436,7 +3436,7 @@ class NSISLexer(RegexLexer):
 
 class RPMSpecLexer(RegexLexer):
     """
-    For RPM *.spec files
+    For RPM ``.spec`` files.
 
     *New in Pygments 1.6.*
     """
diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py
index 740f72a0..c35e60ab 100644
--- a/pygments/lexers/web.py
+++ b/pygments/lexers/web.py
@@ -4173,9 +4173,9 @@ class CirruLexer(RegexLexer):
     Syntax rules of Cirru can be found at:
     http://grammar.cirru.org/
 
-    * using `()` to markup blocks, but limited in the same line
-    * using `""` to markup strings, allow `\` to escape
-    * using `$` as a shorthand for `()` till indentation end or `)`
+    * using ``()`` to markup blocks, but limited in the same line
+    * using ``""`` to markup strings, allow ``\`` to escape
+    * using ``$`` as a shorthand for ``()`` till indentation end or ``)``
     * using indentations for create nesting
 
     *New in Pygments 1.7.*
diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py
new file mode 100644
index 00000000..5ab8f060
--- /dev/null
+++ b/pygments/sphinxext.py
@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.sphinxext
+    ~~~~~~~~~~~~~~~~~~
+
+    Sphinx extension to generate automatic documentation of lexers,
+    formatters and filters.
+
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+import sys
+
+from docutils import nodes
+from docutils.statemachine import ViewList
+from sphinx.util.compat import Directive
+from sphinx.util.nodes import nested_parse_with_titles
+
+
+MODULEDOC = '''
+.. module:: %s
+
+%s
+%s
+'''
+
+LEXERDOC = '''
+.. class:: %s
+
+    :Short names: %s
+    :Filenames:   %s
+    :MIME types:  %s
+
+    %s
+
+'''
+
+FMTERDOC = '''
+.. class:: %s
+
+    :Short names: %s
+    :Filenames: %s
+
+    %s
+
+'''
+
+FILTERDOC = '''
+.. class:: %s
+
+    :Name: %s
+
+    %s
+
+'''
+
+class PygmentsDoc(Directive):
+    """
+    A directive to collect all lexers/formatters/filters and generate
+    autoclass directives for them.
+    """
+    has_content = False
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = False
+    option_spec = {}
+
+    def run(self):
+        self.filenames = set()
+        if self.arguments[0] == 'lexers':
+            out = self.document_lexers()
+        elif self.arguments[0] == 'formatters':
+            out = self.document_formatters()
+        elif self.arguments[0] == 'filters':
+            out = self.document_filters()
+        else:
+            raise Exception('invalid argument for "pygmentsdoc" directive')
+        node = nodes.compound()
+        vl = ViewList(out.split('\n'), source='')
+        nested_parse_with_titles(self.state, vl, node)
+        for fn in self.filenames:
+            self.state.document.settings.record_dependencies.add(fn)
+        return node.children
+
+    def document_lexers(self):
+        from pygments.lexers._mapping import LEXERS
+        out = []
+        modules = {}
+        moduledocstrings = {}
+        for classname, data in sorted(LEXERS.items(), key=lambda x: x[0]):
+            module = data[0]
+            mod = __import__(module, None, None, [classname])
+            self.filenames.add(mod.__file__)
+            cls = getattr(mod, classname)
+            if not cls.__doc__:
+                print("Warning: %s does not have a docstring." % classname)
+            docstring = cls.__doc__
+            if isinstance(docstring, bytes):
+                docstring = docstring.decode('utf8')
+            modules.setdefault(module, []).append((
+                classname,
+                ', '.join(data[2]) or 'None',
+                ', '.join(data[3]).replace('*', '\\*').replace('_', '\\') or 'None',
+                ', '.join(data[4]) or 'None',
+                docstring))
+            if module not in moduledocstrings:
+                moddoc = mod.__doc__
+                if isinstance(moddoc, bytes):
+                    moddoc = moddoc.decode('utf8')
+                moduledocstrings[module] = moddoc
+
+        for module, lexers in sorted(modules.items(), key=lambda x: x[0]):
+            heading = moduledocstrings[module].splitlines()[4].strip().rstrip('.')
+            out.append(MODULEDOC % (module, heading, '-'*len(heading)))
+            for data in lexers:
+                out.append(LEXERDOC % data)
+
+        return ''.join(out)
+
+    def document_formatters(self):
+        from pygments.formatters import FORMATTERS
+
+        out = []
+        for cls, data in sorted(FORMATTERS.items(),
+                                key=lambda x: x[0].__name__):
+            self.filenames.add(sys.modules[cls.__module__].__file__)
+            docstring = cls.__doc__
+            if isinstance(docstring, bytes):
+                docstring = docstring.decode('utf8')
+            heading = cls.__name__
+            out.append(FMTERDOC % (heading, ', '.join(data[1]) or 'None',
+                                   ', '.join(data[2]).replace('*', '\\*') or 'None',
+                                   docstring))
+        return ''.join(out)
+
+    def document_filters(self):
+        from pygments.filters import FILTERS
+
+        out = []
+        for name, cls in FILTERS.items():
+            self.filenames.add(sys.modules[cls.__module__].__file__)
+            docstring = cls.__doc__
+            if isinstance(docstring, bytes):
+                docstring = docstring.decode('utf8')
+            out.append(FILTERDOC % (cls.__name__, name, docstring))
+        return ''.join(out)
+
+
+def setup(app):
+    app.add_directive('pygmentsdoc', PygmentsDoc)