Enter a 'pre-token' state after seeing a token keyword (regex, token, or rule)

This is so that any Perl 6 code between the keyword and the opening brace may
be properly parsed.  Handles code like this:

    token something(@params) { ... }

1 files changed, 6 insertions, 1 deletions

diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py
index f207bcc1..685b622a 100644
--- a/pygments/lexers/agile.py
+++ b/pygments/lexers/agile.py
@@ -2164,7 +2164,7 @@ class Perl6Lexer(ExtendedRegexLexer):
             ( r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline ),
             ( r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline ),
             ( r'^=.*?\n\s*?\n', Comment.Multiline ),
-            ( r'(regex|token|rule)(?![' + PERL6_IDENTIFIER_CHARS + '])(\s*[' + PERL6_IDENTIFIER_CHARS + ']+:sym<.*?>)?(.*?)([{])', bygroups(Keyword, Name, Name, Text), 'token' ),
+            ( r'(regex|token|rule)(?![' + PERL6_IDENTIFIER_CHARS + '])(\s*[' + PERL6_IDENTIFIER_CHARS + ']+:sym<.*?>)?', bygroups(Keyword, Name), 'pre-token' ),
             # deal with a special class in the Perl 6 grammar (role q { ... })
             ( r'(role)(\s*)(q)(\s*)', bygroups(Keyword, Text, Name, Text) ),
             ( _build_word_match(PERL6_KEYWORDS, PERL6_IDENTIFIER_CHARS), Keyword ),
@@ -2201,6 +2201,11 @@ class Perl6Lexer(ExtendedRegexLexer):
             ( r'[}]', closing_brace_callback ),
             ( r'.+?', Text ),
         ],
+        'pre-token' : [
+            include('common'),
+            ( r'[{]', Text, ( '#pop', 'token' ) ),
+            ( r'.+?', Text ),
+        ],
         # the tokens state rules are defined after the class body, for reasons
         # explained below.
     }