[svn] ported the myghty lexer by zzzeek (correct amount of zs and es?) to pygments. Probably licensing issue, i'll resolve that via IRC once he's online

5 files changed, 258 insertions, 3 deletions

diff --git a/docs/src/lexerdevelopment.txt b/docs/src/lexerdevelopment.txt
index 31a884e8..cffb4dd0 100644
--- a/docs/src/lexerdevelopment.txt
+++ b/docs/src/lexerdevelopment.txt
@@ -135,6 +135,11 @@ group (a ``(...)``), and there must not be any nested capturing groups.  If you
 nevertheless need a group, use a non-capturing group defined using this syntax:
 ``r'(?:some|words|here)'`` (note the ``?:`` after the beginning parenthesis).
 
+Sometimes it could be that you need a position group inside the regex which
+shouldn't be part of the output but is used in the regular expressions for
+matching the group twice (eg: ``r'(<(foo|bar)>)(.*?)(</\2>)'``). In such a
+case you can pass `None` to the bygroups function and it will skip that match.
+
 
 Changing states
 ===============
diff --git a/pygments/lexer.py b/pygments/lexer.py
index ffaeeb55..5b5d18ba 100644
--- a/pygments/lexer.py
+++ b/pygments/lexer.py
@@ -241,7 +241,9 @@ def bygroups(*args):
     """
     def callback(lexer, match, ctx=None):
         for i, action in enumerate(args):
-            if type(action) is _TokenType:
+            if action is None:
+                continue
+            elif type(action) is _TokenType:
                 data = match.group(i + 1)
                 if data:
                     yield match.start(i + 1), action, data
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index ede90ac0..68b8be3c 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -47,6 +47,11 @@ LEXERS = {
     'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ()),
     'LuaLexer': ('pygments.lexers.agile', 'Lua', ('lua',), ('*.lua',), ('text/x-lua', 'application/x-lua')),
     'MakefileLexer': ('pygments.lexers.text', 'Makefile', ('make', 'makefile', 'mf'), ('*.mak', 'Makefile', 'makefile'), ('text/x-makefile',)),
+    'MyghtyCssLexer': ('pygments.lexers.templates', 'CSS+Myghty', ('css+myghty',), (), ()),
+    'MyghtyHtmlLexer': ('pygments.lexers.templates', 'HTML+Myghty', ('html+myghty',), (), ()),
+    'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ()),
+    'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ()),
+    'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ()),
     'PerlLexer': ('pygments.lexers.agile', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm'), ('text/x-perl', 'application/x-perl')),
     'PhpLexer': ('pygments.lexers.web', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]'), ()),
     'PythonConsoleLexer': ('pygments.lexers.agile', 'Python console session', ('pycon',), (), ()),
diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py
index 21956e47..0d19e94c 100644
--- a/pygments/lexers/templates.py
+++ b/pygments/lexers/templates.py
@@ -19,7 +19,7 @@ from pygments.lexers.web import \
      PhpLexer, HtmlLexer, XmlLexer, JavascriptLexer, CssLexer
 from pygments.lexers.agile import PythonLexer
 from pygments.lexer import Lexer, DelegatingLexer, RegexLexer, bygroups, \
-     include, using
+     include, using, this
 from pygments.token import Error, \
      Text, Comment, Operator, Keyword, Name, String, Number, Other
 from pygments.util import html_doctype_matches, looks_like_xml
@@ -31,7 +31,9 @@ __all__ = ['HtmlPhpLexer', 'XmlPhpLexer', 'CssPhpLexer',
            'CssSmartyLexer', 'JavascriptSmartyLexer', 'DjangoLexer',
            'HtmlDjangoLexer', 'CssDjangoLexer', 'XmlDjangoLexer',
            'JavascriptDjangoLexer', 'GenshiLexer', 'HtmlGenshiLexer',
-           'GenshiTextLexer', 'CssGenshiLexer', 'JavascriptGenshiLexer']
+           'GenshiTextLexer', 'CssGenshiLexer', 'JavascriptGenshiLexer',
+           'MyghtyLexer', 'MyghtyHtmlLexer', 'MyghtyXmlLexer',
+           'MyghtyCssLexer', 'MyghtyJavascriptLexer']
 
 
 class ErbLexer(Lexer):
@@ -221,6 +223,81 @@ class DjangoLexer(RegexLexer):
         return rv
 
 
+class MyghtyLexer(RegexLexer):
+    name = 'Myghty'
+    aliases = ['myghty']
+    filenames = ['*.myt', 'autodelegate']
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'(<%(def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)(?s)',
+             bygroups(Name.Tag, None, Text, Name.Function, Name.Tag,
+                      using(this), Name.Tag)),
+            (r'(<%(\w+))(.*?)(>)(.*?)(</%\2\s*>)(?s)',
+             bygroups(Name.Tag, None, Name.Function, Name.Tag,
+                      using(PythonLexer), Name.Tag)),
+            (r'(<&[^|])(.*?)(,.*?)?(&>)',
+             bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)),
+            (r'(<&\|)(.*?)(,.*?)?(&>)(?s)',
+             bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)),
+            (r'</&>', Name.Tag),
+            (r'(<%)(.*?)(%>)',
+             bygroups(Name.Tag, using(PythonLexer), Name.Tag)),
+            (r'(?<=^)\#[^\n]*(\n|\Z)', Comment),
+            (r'(?<=^)(\%)([^\n]*)(\n|\Z)',
+             bygroups(Name.Tag, using(PythonLexer), Other)),
+            (r'''(?sx)
+                (.+?)               # anything, followed by:
+                (?:
+                 (?<=\n)(?=[%#]) |  # an eval or comment line 
+                 (?=</?[%&]) |      # a substitution or block or
+                                    # call start or end
+                                    # - don't consume
+                 (\\\n) |           # an escaped newline  
+                 \Z                 # end of string
+                )
+            ''', bygroups(Other, Operator)),
+        ]
+    }
+
+
+class MyghtyHtmlLexer(DelegatingLexer):
+    name = 'HTML+Myghty'
+    aliases = ['html+myghty']
+
+    def __init__(self, **options):
+        super(MyghtyHtmlLexer, self).__init__(HtmlLexer, MyghtyLexer,
+                                              **options)
+
+
+class MyghtyXmlLexer(DelegatingLexer):
+    name = 'XML+Myghty'
+    aliases = ['xml+myghty']
+
+    def __init__(self, **options):
+        super(MyghtyXmlLexer, self).__init__(XmlLexer, MyghtyLexer,
+                                             **options)
+
+
+class MyghtyJavascriptLexer(DelegatingLexer):
+    name = 'JavaScript+Myghty'
+    aliases = ['js+myghty', 'javascript+myghty']
+
+    def __init__(self, **options):
+        super(MyghtyJavascriptLexer, self).__init__(JavascriptLexer,
+                                                    MyghtyLexer, **options)
+
+
+class MyghtyCssLexer(DelegatingLexer):
+    name = 'CSS+Myghty'
+    aliases = ['css+myghty']
+
+    def __init__(self, **options):
+        super(MyghtyCssLexer, self).__init__(CssLexer, MyghtyLexer,
+                                             **options)
+
+
 # Genshi lexers courtesy of Matt Good.
 
 class GenshiTextLexer(RegexLexer):
diff --git a/tests/examplefiles/test.myt b/tests/examplefiles/test.myt
new file mode 100644
index 00000000..1668f7a6
--- /dev/null
+++ b/tests/examplefiles/test.myt
@@ -0,0 +1,166 @@
+<%doc>formatting.myt - Provides section formatting elements, syntax-highlighted code blocks, and other special filters.</%doc>
+
+<%global>
+    import string, re
+    import highlight
+</%global>
+
+<%method section>
+<%doc>Main section formatting element.</%doc>
+<%args>
+    toc
+    path
+    description=None
+    onepage=False
+</%args>
+<%init>
+    item = toc.get_by_path(path)
+    if item is None:
+        raise "path: " + path
+</%init>
+
+<A name="<% item.path %>"></a>
+
+<div class="subsection" style="margin-left:<% repr(item.depth * 10) %>px;">
+
+<%python>
+    content = m.content()
+    re2 = re.compile(r"'''PYESC(.+?)PYESC'''", re.S)
+    content = re2.sub(lambda m: m.group(1), content)
+</%python>
+
+% if item.depth > 1:
+<h3><% description or item.description %></h3>
+%
+
+    <div class="sectiontext">
+    <% content %>
+    </div>
+
+% if onepage or item.depth > 1:
+%   if (item.next and item.next.depth >= item.depth):
+    <a href="#<% item.get_page_root().path %>" class="toclink">back to section top</a>
+%
+% else:
+    <a href="#<% item.get_page_root().path %>" class="toclink">back to section top</a>
+    <& nav.myt:pagenav, item=item, onepage=onepage &>
+% 
+</div>
+
+</%method>
+
+
+<%method formatplain>
+    <%filter>
+        import re
+        f = re.sub(r'\n[\s\t]*\n[\s\t]*', '</p>\n<p>', f)
+        f = "<p>" + f + "</p>"
+        return f
+    </%filter>
+<% m.content() | h%>
+</%method>
+
+
+
+
+<%method codeline trim="both">
+<span class="codeline"><% m.content() %></span>
+</%method>
+
+<%method code autoflush=False>
+<%args>
+    title = None
+    syntaxtype = 'python'
+    html_escape = False
+    use_sliders = False
+</%args>
+
+<%init>
+    def fix_indent(f):
+        f =string.expandtabs(f, 4)
+        g = ''
+        lines = string.split(f, "\n")
+        whitespace = None
+        for line in lines:
+            if whitespace is None:
+                match = re.match(r"^([ ]*).+", line)
+                if match is not None:
+                    whitespace = match.group(1)
+
+            if whitespace is not None:
+                line = re.sub(r"^%s" % whitespace, "", line)
+
+            if whitespace is not None or re.search(r"\w", line) is not None:
+                g += (line + "\n")
+
+
+        return g.rstrip()
+
+    p = re.compile(r'<pre>(.*?)</pre>', re.S)
+    def hlight(match):
+        return "<pre>" + highlight.highlight(fix_indent(match.group(1)), html_escape = html_escape, syntaxtype = syntaxtype) + "</pre>"
+    content = p.sub(hlight, "<pre>" + m.content() + "</pre>")
+</%init>
+<div class="<% use_sliders and "sliding_code" or "code" %>">
+% if title is not None:
+    <div class="codetitle"><% title %></div>
+%
+<% content %></div>
+</%method>
+
+
+
+
+<%method popboxlink trim="both"> 
+    <%args>
+        name=None
+        show='show'
+        hide='hide'
+    </%args>
+    <%init>
+        if name is None:
+            name = m.attributes.setdefault('popbox_name', 0)
+        name += 1
+        m.attributes['popbox_name'] = name
+        name = "popbox_" + repr(name)
+    </%init>
+javascript:togglePopbox('<% name %>', '<% show %>', '<% hide %>')
+</%method>
+
+<%method popbox trim="both">
+<%args>
+    name = None
+    class_ = None
+</%args>
+<%init>
+    if name is None:
+        name = 'popbox_' + repr(m.attributes['popbox_name'])
+</%init>
+<div id="<% name %>_div" class="<% class_ %>" style="display:none;"><% m.content().strip() %></div>
+</%method>
+
+<%method poplink trim="both">
+    <%args>
+        link='sql'
+    </%args>
+    <%init>
+        href = m.scomp('SELF:popboxlink')
+    </%init>
+    '''PYESC<& nav.myt:link, href=href, text=link, class_="codepoplink" &>PYESC'''
+</%method>
+
+<%method codepopper trim="both">
+	<%init>
+		c = m.content()
+		c = re.sub(r'\n', '<br/>\n', c.strip())
+	</%init>
+    </pre><&|SELF:popbox, class_="codepop" &><% c %></&><pre>
+</%method>
+
+<%method poppedcode trim="both">
+	<%init>
+		c = m.content()
+		c = re.sub(r'\n', '<br/>\n', c.strip())
+	</%init>
+    </pre><div class="codepop"><% c %></div><pre>
+</%method>