Add a lexer for JSGF

4 files changed, 115 insertions, 5 deletions

diff --git a/AUTHORS b/AUTHORS
index 9318d438..63352803 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -36,7 +36,7 @@ Other contributors, listed alphabetically, are:
 * chebee7i -- Python traceback lexer improvements
 * Hiram Chirino -- Scaml and Jade lexers
 * Ian Cooper -- VGL lexer
-* David Corbett -- Inform, Jasmin, and TADS 3 lexers
+* David Corbett -- Inform, Jasmin, JSGF, and TADS 3 lexers
 * Leaf Corcoran -- MoonScript lexer
 * Christopher Creutzig -- MuPAD lexer
 * Daniël W. Crompton -- Pike lexer
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 3afa2692..53033649 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -200,6 +200,7 @@ LEXERS = {
     'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
     'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
     'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)),
+    'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')),
     'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
     'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)),
     'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
diff --git a/pygments/lexers/grammar_notation.py b/pygments/lexers/grammar_notation.py
index 460914f4..21c59b09 100644
--- a/pygments/lexers/grammar_notation.py
+++ b/pygments/lexers/grammar_notation.py
@@ -9,11 +9,13 @@
     :license: BSD, see LICENSE for details.
 """
 
-from pygments.lexer import RegexLexer, bygroups, words
-from pygments.token import Punctuation, Text, Comment, Operator, \
-    Keyword, Name, Literal
+import re
 
-__all__ = ['BnfLexer', 'AbnfLexer']
+from pygments.lexer import RegexLexer, bygroups, include, this, using, words
+from pygments.token import Comment, Keyword, Literal, Name, Number, \
+     Operator, Punctuation, String, Text
+
+__all__ = ['BnfLexer', 'AbnfLexer', 'JsgfLexer']
 
 
 class BnfLexer(RegexLexer):
@@ -129,3 +131,82 @@ class AbnfLexer(RegexLexer):
             (r'.', Text),
         ],
     }
+
+
+class JsgfLexer(RegexLexer):
+    """
+    For `JSpeech Grammar Format <https://www.w3.org/TR/jsgf/>`_
+    grammars.
+    """
+
+    name = 'JSGF'
+    aliases = ['jsgf']
+    filenames = ['*.jsgf']
+    mimetypes = ['application/jsgf', 'application/x-jsgf', 'text/jsgf']
+
+    flags = re.MULTILINE | re.UNICODE
+
+    tokens = {
+        'root': [
+            include('comments'),
+            include('non-comments'),
+        ],
+        'comments': [
+            (r'/\*\*(?!/)', Comment.Multiline, 'documentation comment'),
+            (r'/\*[\w\W]*?\*/', Comment.Multiline),
+            (r'//.*', Comment.Single),
+        ],
+        'non-comments': [
+            ('\A#JSGF[^;]*', Comment.Preproc),
+            (r'\s+', Text),
+            (r';', Punctuation),
+            (r'[=|()\[\]*+]', Operator),
+            (r'/[^/]+/', Number.Float),
+            (r'"', String.Double, 'string'),
+            (r'\{', String.Other, 'tag'),
+            (words(('import', 'public'), suffix=r'\b'), Keyword.Reserved),
+            (r'grammar\b', Keyword.Reserved, 'grammar name'),
+            (r'(<)(NULL|VOID)(>)',
+             bygroups(Punctuation, Name.Builtin, Punctuation)),
+            (r'<', Punctuation, 'rulename'),
+            (r'\w+|[^\s;=|()\[\]*+/"{<\w]+', Text),
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'\\.', String.Escape),
+            (r'[^\\"]+', String.Double),
+        ],
+        'tag': [
+            (r'\}', String.Other, '#pop'),
+            (r'\\.', String.Escape),
+            (r'[^\\}]+', String.Other),
+        ],
+        'grammar name': [
+            (r';', Punctuation, '#pop'),
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[^;\s.]+', Name.Namespace),
+        ],
+        'rulename': [
+            (r'>', Punctuation, '#pop'),
+            (r'\*', Punctuation),
+            (r'\s+', Text),
+            (r'([^.>]+)(\s*)(\.)', bygroups(Name.Namespace, Text, Punctuation)),
+            (r'[^.>]+', Name.Constant),
+        ],
+        'documentation comment': [
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'(^\s*\*?\s*)(@(?:example|see)\s+)'
+             r'([\w\W]*?(?=(?:^\s*\*?\s*@|\*/)))',
+             bygroups(Comment.Multiline, Comment.Special,
+                      using(this, state='example'))),
+            (r'(^\s*\*?\s*)(@\S*)',
+             bygroups(Comment.Multiline, Comment.Special)),
+            (r'[^*\n@]+|\w|\W', Comment.Multiline),
+        ],
+        'example': [
+            (r'\n\s*\*', Comment.Multiline),
+            include('non-comments'),
+            (r'.', Comment.Multiline),
+        ],
+    }
diff --git a/tests/examplefiles/example.jsgf b/tests/examplefiles/example.jsgf
new file mode 100644
index 00000000..cd79dbed
--- /dev/null
+++ b/tests/examplefiles/example.jsgf
@@ -0,0 +1,28 @@
+#JSGF V1.0 UTF-8 en;

+

+grammar org.pygments.example;

+

+// comment /*

+/*

+  multi-line

+  comment

+  /* without nesting

+  @example doesn't mean anything here.

+*/

+/**/

+

+/**

+  * A silly @example grammar.

+  *

+  * @author David Corbett

+  * @version 1

+* @see <org.pygments.example.en>

+  * @example hello world

+    @example hello ","

+  *

+  **@blah

+  **world

+  */

+public <sentence> = (/1/<en> | / 0.8f /<fr> |/0/<VOID>""""{}{})*<NULL>;

+<org.pygments.example.fr> = bonjour {hello} [,] "le monde" {world};

+<en> = (/56/hello|/3.14e3/"\"hello\"") {hello} {{ {\\//\} } world {world} !+ ;