From 36d94127527aeb6eb76c77cd56ae5fa9088e02e6 Mon Sep 17 00:00:00 2001 From: David Corbett Date: Sun, 24 Jan 2016 10:17:34 -0500 Subject: Add a lexer for JSGF --- AUTHORS | 2 +- pygments/lexers/_mapping.py | 1 + pygments/lexers/grammar_notation.py | 89 +++++++++++++++++++++++++++++++++++-- tests/examplefiles/example.jsgf | 28 ++++++++++++ 4 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 tests/examplefiles/example.jsgf diff --git a/AUTHORS b/AUTHORS index 9318d438..63352803 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,7 +36,7 @@ Other contributors, listed alphabetically, are: * chebee7i -- Python traceback lexer improvements * Hiram Chirino -- Scaml and Jade lexers * Ian Cooper -- VGL lexer -* David Corbett -- Inform, Jasmin, and TADS 3 lexers +* David Corbett -- Inform, Jasmin, JSGF, and TADS 3 lexers * Leaf Corcoran -- MoonScript lexer * Christopher Creutzig -- MuPAD lexer * Daniël W. Crompton -- Pike lexer diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 3afa2692..53033649 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -200,6 +200,7 @@ LEXERS = { 'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')), 'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')), 'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)), + 'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')), 'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)), 'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)), 'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)), diff --git a/pygments/lexers/grammar_notation.py b/pygments/lexers/grammar_notation.py index 460914f4..21c59b09 100644 --- a/pygments/lexers/grammar_notation.py +++ b/pygments/lexers/grammar_notation.py @@ -9,11 +9,13 @@ :license: BSD, see LICENSE for details. """ -from pygments.lexer import RegexLexer, bygroups, words -from pygments.token import Punctuation, Text, Comment, Operator, \ - Keyword, Name, Literal +import re -__all__ = ['BnfLexer', 'AbnfLexer'] +from pygments.lexer import RegexLexer, bygroups, include, this, using, words +from pygments.token import Comment, Keyword, Literal, Name, Number, \ + Operator, Punctuation, String, Text + +__all__ = ['BnfLexer', 'AbnfLexer', 'JsgfLexer'] class BnfLexer(RegexLexer): @@ -129,3 +131,82 @@ class AbnfLexer(RegexLexer): (r'.', Text), ], } + + +class JsgfLexer(RegexLexer): + """ + For `JSpeech Grammar Format `_ + grammars. + """ + + name = 'JSGF' + aliases = ['jsgf'] + filenames = ['*.jsgf'] + mimetypes = ['application/jsgf', 'application/x-jsgf', 'text/jsgf'] + + flags = re.MULTILINE | re.UNICODE + + tokens = { + 'root': [ + include('comments'), + include('non-comments'), + ], + 'comments': [ + (r'/\*\*(?!/)', Comment.Multiline, 'documentation comment'), + (r'/\*[\w\W]*?\*/', Comment.Multiline), + (r'//.*', Comment.Single), + ], + 'non-comments': [ + ('\A#JSGF[^;]*', Comment.Preproc), + (r'\s+', Text), + (r';', Punctuation), + (r'[=|()\[\]*+]', Operator), + (r'/[^/]+/', Number.Float), + (r'"', String.Double, 'string'), + (r'\{', String.Other, 'tag'), + (words(('import', 'public'), suffix=r'\b'), Keyword.Reserved), + (r'grammar\b', Keyword.Reserved, 'grammar name'), + (r'(<)(NULL|VOID)(>)', + bygroups(Punctuation, Name.Builtin, Punctuation)), + (r'<', Punctuation, 'rulename'), + (r'\w+|[^\s;=|()\[\]*+/"{<\w]+', Text), + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'\\.', String.Escape), + (r'[^\\"]+', String.Double), + ], + 'tag': [ + (r'\}', String.Other, '#pop'), + (r'\\.', String.Escape), + (r'[^\\}]+', String.Other), + ], + 'grammar name': [ + (r';', Punctuation, '#pop'), + (r'\s+', Text), + (r'\.', Punctuation), + (r'[^;\s.]+', Name.Namespace), + ], + 'rulename': [ + (r'>', Punctuation, '#pop'), + (r'\*', Punctuation), + (r'\s+', Text), + (r'([^.>]+)(\s*)(\.)', bygroups(Name.Namespace, Text, Punctuation)), + (r'[^.>]+', Name.Constant), + ], + 'documentation comment': [ + (r'\*/', Comment.Multiline, '#pop'), + (r'(^\s*\*?\s*)(@(?:example|see)\s+)' + r'([\w\W]*?(?=(?:^\s*\*?\s*@|\*/)))', + bygroups(Comment.Multiline, Comment.Special, + using(this, state='example'))), + (r'(^\s*\*?\s*)(@\S*)', + bygroups(Comment.Multiline, Comment.Special)), + (r'[^*\n@]+|\w|\W', Comment.Multiline), + ], + 'example': [ + (r'\n\s*\*', Comment.Multiline), + include('non-comments'), + (r'.', Comment.Multiline), + ], + } diff --git a/tests/examplefiles/example.jsgf b/tests/examplefiles/example.jsgf new file mode 100644 index 00000000..cd79dbed --- /dev/null +++ b/tests/examplefiles/example.jsgf @@ -0,0 +1,28 @@ +#JSGF V1.0 UTF-8 en; + +grammar org.pygments.example; + +// comment /* +/* + multi-line + comment + /* without nesting + @example doesn't mean anything here. +*/ +/**/ + +/** + * A silly @example grammar. + * + * @author David Corbett + * @version 1 +* @see + * @example hello world + @example hello "," + * + **@blah + **world + */ +public = (/1/ | / 0.8f / |/0/""""{}{})*; + = bonjour {hello} [,] "le monde" {world}; + = (/56/hello|/3.14e3/"\"hello\"") {hello} {{ {\\//\} } world {world} !+ ; -- cgit v1.2.1