Refactored formatter mapping to work like the lexer mapping.

Thanks to Ilia Choly for the initial pull request.

6 files changed, 165 insertions, 140 deletions

diff --git a/Makefile b/Makefile
index e28c90c7..9dcbe9d9 100644
--- a/Makefile
+++ b/Makefile
@@ -40,8 +40,8 @@ docs:
 	make -C doc html
 
 mapfiles:
-	(cd pygments/lexers; $(PYTHON) _mapping.py)
 	(cd pygments/formatters; $(PYTHON) _mapping.py)
+	(cd pygments/lexers; $(PYTHON) _mapping.py)
 
 pylint:
 	@pylint --rcfile scripts/pylintrc pygments
diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py
index f0c5dc41..45ceaff6 100644
--- a/pygments/formatters/__init__.py
+++ b/pygments/formatters/__init__.py
@@ -8,63 +8,111 @@
     :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-import os.path
-import fnmatch
+
 import re
+import sys
+import types
+import fnmatch
+from os.path import basename
 
 from pygments.formatters._mapping import FORMATTERS
 from pygments.plugin import find_plugin_formatters
-from pygments.util import ClassNotFound
-
-ns = globals()
-for fcls in FORMATTERS:
-    ns[fcls.__name__] = fcls
-del fcls
+from pygments.util import ClassNotFound, itervalues
 
 __all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
-           'get_all_formatters'] + [cls.__name__ for cls in FORMATTERS]
+           'get_all_formatters'] + list(FORMATTERS)
+
+_formatter_cache = {}  # classes by name
+_pattern_cache = {}
+
+
+def _fn_matches(fn, glob):
+    """Return whether the supplied file name fn matches pattern filename."""
+    if glob not in _pattern_cache:
+        pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
+        return pattern.match(fn)
+    return _pattern_cache[glob].match(fn)
+
+
+def _load_formatters(module_name):
+    """Load a formatter (and all others in the module too)."""
+    mod = __import__(module_name, None, None, ['__all__'])
+    for formatter_name in mod.__all__:
+        cls = getattr(mod, formatter_name)
+        _formatter_cache[cls.name] = cls
+
 
+def get_all_formatters():
+    """Return a generator for all formatter classes."""
+    # NB: this returns formatter classes, not info like get_all_lexers().
+    for info in itervalues(FORMATTERS):
+        if info[1] not in _formatter_cache:
+            _load_formatters(info[0])
+        yield _formatter_cache[info[1]]
+    for _, formatter in find_plugin_formatters():
+        yield formatter
 
-_formatter_alias_cache = {}
-_formatter_filename_cache = []
 
-def _init_formatter_cache():
-    if _formatter_alias_cache:
-        return
-    for cls in get_all_formatters():
-        for alias in cls.aliases:
-            _formatter_alias_cache[alias] = cls
-        for fn in cls.filenames:
-            _formatter_filename_cache.append((
-                re.compile(fnmatch.translate(fn)), cls))
+def find_formatter_class(alias):
+    """Lookup a formatter by alias.
 
+    Returns None if not found.
+    """
+    for module_name, name, aliases, _, _ in itervalues(FORMATTERS):
+        if alias in aliases:
+            if name not in _formatter_cache:
+                _load_formatters(module_name)
+            return _formatter_cache[name]
+    for _, cls in find_plugin_formatters():
+        if alias in cls.aliases:
+            return cls
 
-def find_formatter_class(name):
-    _init_formatter_cache()
-    cls = _formatter_alias_cache.get(name, None)
-    return cls
 
+def get_formatter_by_name(_alias, **options):
+    """Lookup and instantiate a formatter by alias.
 
-def get_formatter_by_name(name, **options):
-    _init_formatter_cache()
-    cls = _formatter_alias_cache.get(name, None)
-    if not cls:
-        raise ClassNotFound("No formatter found for name %r" % name)
+    Raises ClassNotFound if not found.
+    """
+    cls = find_formatter_class(_alias)
+    if cls is None:
+        raise ClassNotFound("No formatter found for name %r" % _alias)
     return cls(**options)
 
 
 def get_formatter_for_filename(fn, **options):
-    _init_formatter_cache()
-    fn = os.path.basename(fn)
-    for pattern, cls in _formatter_filename_cache:
-        if pattern.match(fn):
-            return cls(**options)
+    """Lookup and instantiate a formatter by filename pattern.
+
+    Raises ClassNotFound if not found.
+    """
+    fn = basename(fn)
+    for modname, name, _, filenames, _ in itervalues(FORMATTERS):
+        for filename in filenames:
+            if _fn_matches(fn, filename):
+                if name not in _formatter_cache:
+                    _load_formatters(modname)
+                return _formatter_cache[name](**options)
+    for cls in find_plugin_formatters():
+        for filename in cls.filenames:
+            if _fn_matches(fn, filename):
+                return cls(**options)
     raise ClassNotFound("No formatter found for file name %r" % fn)
 
 
-def get_all_formatters():
-    """Return a generator for all formatters."""
-    for formatter in FORMATTERS:
-        yield formatter
-    for _, formatter in find_plugin_formatters():
-        yield formatter
+class _automodule(types.ModuleType):
+    """Automatically import formatters."""
+
+    def __getattr__(self, name):
+        info = FORMATTERS.get(name)
+        if info:
+            _load_formatters(info[0])
+            cls = _formatter_cache[info[1]]
+            setattr(self, name, cls)
+            return cls
+        raise AttributeError(name)
+
+
+oldmod = sys.modules[__name__]
+newmod = _automodule(__name__)
+newmod.__dict__.update(oldmod.__dict__)
+sys.modules[__name__] = newmod
+del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types
diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py
index 8b3fc977..1a083904 100755
--- a/pygments/formatters/_mapping.py
+++ b/pygments/formatters/_mapping.py
@@ -14,45 +14,22 @@
 """
 
 from __future__ import print_function
-try:
-    import pygments
-except ImportError:
-    # This block makes this mapping work like the lexer one -- not requiring
-    # that Pygments already be on your PYTHONPATH.
-    import os.path, sys
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
-
-# start
-from pygments.formatters.bbcode import BBCodeFormatter
-from pygments.formatters.html import HtmlFormatter
-from pygments.formatters.img import BmpImageFormatter
-from pygments.formatters.img import GifImageFormatter
-from pygments.formatters.img import ImageFormatter
-from pygments.formatters.img import JpgImageFormatter
-from pygments.formatters.latex import LatexFormatter
-from pygments.formatters.other import NullFormatter
-from pygments.formatters.other import RawTokenFormatter
-from pygments.formatters.other import TestcaseFormatter
-from pygments.formatters.rtf import RtfFormatter
-from pygments.formatters.svg import SvgFormatter
-from pygments.formatters.terminal import TerminalFormatter
-from pygments.formatters.terminal256 import Terminal256Formatter
 
 FORMATTERS = {
-    BBCodeFormatter: ('BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
-    BmpImageFormatter: ('img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
-    GifImageFormatter: ('img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
-    HtmlFormatter: ('HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
-    ImageFormatter: ('img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
-    JpgImageFormatter: ('img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
-    LatexFormatter: ('LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
-    NullFormatter: ('Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'),
-    RawTokenFormatter: ('Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
-    RtfFormatter: ('RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft\xc2\xae Word\xc2\xae documents.'),
-    SvgFormatter: ('SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
-    Terminal256Formatter: ('Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
-    TerminalFormatter: ('Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
-    TestcaseFormatter: ('Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
+    'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
+    'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
+    'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
+    'NullFormatter': ('pygments.formatters.other', 'Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'),
+    'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
+    'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft\xc2\xae Word\xc2\xae documents.'),
+    'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
+    'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
 }
 
 if __name__ == '__main__':
@@ -65,23 +42,24 @@ if __name__ == '__main__':
     sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
     from pygments.util import docstring_headline
 
-    for filename in os.listdir('.'):
-        if filename.endswith('.py') and not filename.startswith('_'):
-            module_name = 'pygments.formatters.%s' % filename[:-3]
-            print(module_name)
-            module = __import__(module_name, None, None, [''])
-            for formatter_name in module.__all__:
-                imports.append((module_name, formatter_name))
-                formatter = getattr(module, formatter_name)
-                found_formatters.append(
-                    '%s: %r' % (formatter_name,
-                                (formatter.name,
-                                 tuple(formatter.aliases),
-                                 tuple(formatter.filenames),
-                                 docstring_headline(formatter))))
-    # sort them, that should make the diff files for svn smaller
+    for root, dirs, files in os.walk('.'):
+        for filename in files:
+            if filename.endswith('.py') and not filename.startswith('_'):
+                module_name = 'pygments.formatters%s.%s' % (
+                    root[1:].replace('/', '.'), filename[:-3])
+                print(module_name)
+                module = __import__(module_name, None, None, [''])
+                for formatter_name in module.__all__:
+                    formatter = getattr(module, formatter_name)
+                    found_formatters.append(
+                        '%r: %r' % (formatter_name,
+                                    (module_name,
+                                     formatter.name,
+                                     tuple(formatter.aliases),
+                                     tuple(formatter.filenames),
+                                     docstring_headline(formatter))))
+    # sort them to make the diff minimal
     found_formatters.sort()
-    imports.sort()
 
     # extract useful sourcecode from this file
     f = open(__file__)
@@ -89,15 +67,14 @@ if __name__ == '__main__':
         content = f.read()
     finally:
         f.close()
-    header = content[:content.find('# start')]
+    header = content[:content.find('FORMATTERS = {')]
     footer = content[content.find("if __name__ == '__main__':"):]
 
     # write new file
     f = open(__file__, 'w')
     f.write(header)
-    f.write('# start\n')
-    f.write('\n'.join(['from %s import %s' % imp for imp in imports]))
-    f.write('\n\n')
     f.write('FORMATTERS = {\n    %s\n}\n\n' % ',\n    '.join(found_formatters))
     f.write(footer)
     f.close()
+
+    print ('=== %d formatters processed.' % len(found_formatters))
diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py
index caedd479..576ee4de 100644
--- a/pygments/lexers/__init__.py
+++ b/pygments/lexers/__init__.py
@@ -9,10 +9,10 @@
     :license: BSD, see LICENSE for details.
 """
 
+import re
 import sys
 import types
 import fnmatch
-import re
 from os.path import basename
 
 from pygments.lexers._mapping import LEXERS
@@ -27,23 +27,17 @@ __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
 _lexer_cache = {}
 _pattern_cache = {}
 
+
 def _fn_matches(fn, glob):
-    """
-    Return whether the supplied file name fn matches pattern filename
-    """
+    """Return whether the supplied file name fn matches pattern filename."""
     if glob not in _pattern_cache:
-        pattern = re.compile(fnmatch.translate(glob))
-        _pattern_cache[glob] = pattern
-    else:
-        pattern = _pattern_cache[glob]
-
-    return pattern.match(fn)
+        pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
+        return pattern.match(fn)
+    return _pattern_cache[glob].match(fn)
 
 
 def _load_lexers(module_name):
-    """
-    Load a lexer (and all others in the module too).
-    """
+    """Load a lexer (and all others in the module too)."""
     mod = __import__(module_name, None, None, ['__all__'])
     for lexer_name in mod.__all__:
         cls = getattr(mod, lexer_name)
@@ -51,8 +45,7 @@ def _load_lexers(module_name):
 
 
 def get_all_lexers():
-    """
-    Return a generator of tuples in the form ``(name, aliases,
+    """Return a generator of tuples in the form ``(name, aliases,
     filenames, mimetypes)`` of all know lexers.
     """
     for item in itervalues(LEXERS):
@@ -62,8 +55,9 @@ def get_all_lexers():
 
 
 def find_lexer_class(name):
-    """
-    Lookup a lexer class by name. Return None if not found.
+    """Lookup a lexer class by name.
+
+    Return None if not found.
     """
     if name in _lexer_cache:
         return _lexer_cache[name]
@@ -79,8 +73,9 @@ def find_lexer_class(name):
 
 
 def get_lexer_by_name(_alias, **options):
-    """
-    Get a lexer by an alias.
+    """Get a lexer by an alias.
+
+    Raises ClassNotFound if not found.
     """
     if not _alias:
         raise ClassNotFound('no lexer for alias %r found' % _alias)
@@ -99,10 +94,12 @@ def get_lexer_by_name(_alias, **options):
 
 
 def get_lexer_for_filename(_fn, code=None, **options):
-    """
-    Get a lexer for a filename.  If multiple lexers match the filename
-    pattern, use ``analyse_text()`` to figure out which one is more
-    appropriate.
+    """Get a lexer for a filename.
+
+    If multiple lexers match the filename pattern, use ``analyse_text()`` to
+    figure out which one is more appropriate.
+
+    Raises ClassNotFound if not found.
     """
     matches = []
     fn = basename(_fn)
@@ -141,8 +138,9 @@ def get_lexer_for_filename(_fn, code=None, **options):
 
 
 def get_lexer_for_mimetype(_mime, **options):
-    """
-    Get a lexer for a mimetype.
+    """Get a lexer for a mimetype.
+
+    Raises ClassNotFound if not found.
     """
     for modname, name, _, _, mimetypes in itervalues(LEXERS):
         if _mime in mimetypes:
@@ -156,9 +154,7 @@ def get_lexer_for_mimetype(_mime, **options):
 
 
 def _iter_lexerclasses():
-    """
-    Return an iterator over all lexer classes.
-    """
+    """Return an iterator over all lexer classes."""
     for key in sorted(LEXERS):
         module_name, name = LEXERS[key][:2]
         if name not in _lexer_cache:
@@ -218,9 +214,7 @@ def guess_lexer_for_filename(_fn, _text, **options):
 
 
 def guess_lexer(_text, **options):
-    """
-    Guess a lexer by strong distinctions in the text (eg, shebang).
-    """
+    """Guess a lexer by strong distinctions in the text (eg, shebang)."""
 
     # try to get a vim modeline first
     ft = get_filetype_from_buffer(_text)
@@ -256,8 +250,8 @@ class _automodule(types.ModuleType):
         raise AttributeError(name)
 
 
-oldmod = sys.modules['pygments.lexers']
-newmod = _automodule('pygments.lexers')
+oldmod = sys.modules[__name__]
+newmod = _automodule(__name__)
 newmod.__dict__.update(oldmod.__dict__)
-sys.modules['pygments.lexers'] = newmod
+sys.modules[__name__] = newmod
 del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index a11d9706..d6e054b5 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -383,7 +383,7 @@ if __name__ == '__main__':
                                      tuple(lexer.aliases),
                                      tuple(lexer.filenames),
                                      tuple(lexer.mimetypes))))
-    # sort them, that should make the diff files for svn smaller
+    # sort them to make the diff minimal
     found_lexers.sort()
 
     # extract useful sourcecode from this file
@@ -396,8 +396,10 @@ if __name__ == '__main__':
     footer = content[content.find("if __name__ == '__main__':"):]
 
     # write new file
-    f = open(__file__, 'wb')
+    f = open(__file__, 'w')
     f.write(header)
     f.write('LEXERS = {\n    %s,\n}\n\n' % ',\n    '.join(found_lexers))
     f.write(footer)
     f.close()
+
+    print ('=== %d lexers processed.' % len(found_lexers))
diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py
index 893fa90c..e1f51d62 100644
--- a/tests/test_basic_api.py
+++ b/tests/test_basic_api.py
@@ -151,10 +151,10 @@ def test_formatter_public_api():
     out = StringIO()
     # test that every formatter class has the correct public API
     def verify(formatter, info):
-        assert len(info) == 4
-        assert info[0], "missing formatter name"
-        assert info[1], "missing formatter aliases"
-        assert info[3], "missing formatter docstring"
+        assert len(info) == 5
+        assert info[1], "missing formatter name"
+        assert info[2], "missing formatter aliases"
+        assert info[4], "missing formatter docstring"
 
         if formatter.name == 'Raw tokens':
             # will not work with Unicode output file
@@ -172,7 +172,9 @@ def test_formatter_public_api():
         inst.format(ts, out)
 
     for formatter, info in formatters.FORMATTERS.items():
-        yield verify, formatter, info
+        fmter = getattr(formatters, formatter)
+        yield verify, fmter, info
+
 
 def test_formatter_encodings():
     from pygments.formatters import HtmlFormatter
@@ -223,7 +225,9 @@ def test_formatter_unicode_handling():
             assert type(out) is bytes, '%s: %r' % (formatter, out)
 
     for formatter, info in formatters.FORMATTERS.items():
-        yield verify, formatter
+        # this tests the automatic importing as well
+        fmter = getattr(formatters, formatter)
+        yield verify, fmter
 
 
 def test_get_formatters():