Add World of Warcraft TOC file lexer (#2244)

Also fix a broken link and decode as UTF8 in count_token_references.py.

12 files changed, 771 insertions, 2 deletions

diff --git a/Contributing.md b/Contributing.md
index 0d831f0b..93da428c 100644
--- a/Contributing.md
+++ b/Contributing.md
@@ -11,7 +11,7 @@ Goals & non-goals of Pygments
 Python support
 --------------
 
-Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/#status-of-python-branches). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments.
+Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/versions/). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments.
 
 Validation
 ----------
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 22c70738..540b3f93 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -523,6 +523,7 @@ LEXERS = {
     'WatLexer': ('pygments.lexers.webassembly', 'WebAssembly', ('wast', 'wat'), ('*.wat', '*.wast'), ()),
     'WebIDLLexer': ('pygments.lexers.webidl', 'Web IDL', ('webidl',), ('*.webidl',), ()),
     'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)),
+    'WoWTocLexer': ('pygments.lexers.wowtoc', 'World of Warcraft TOC', ('wowtoc',), ('*.toc',), ()),
     'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)),
     'XMLUL4Lexer': ('pygments.lexers.ul4', 'XML+UL4', ('xml+ul4',), ('*.xmlul4',), ()),
     'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')),
diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py
new file mode 100644
index 00000000..615493cd
--- /dev/null
+++ b/pygments/lexers/wowtoc.py
@@ -0,0 +1,114 @@
+"""
+    pygments.lexers.wowtoc
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for World of Warcraft TOC files, which describe game addon metadata.
+
+    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import re
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Comment, Name, Text, Punctuation, String, Keyword
+
+__all__ = ["WoWTocLexer"]
+
+def _create_tag_line_pattern(inner_pattern, ignore_case=False):
+    return ((r"(?i)" if ignore_case else r"")
+        + r"^(##)( *)"  # groups 1, 2
+        + inner_pattern  # group 3
+        + r"( *)(:)( *)(.*?)( *)$")  # groups 4, 5, 6, 7, 8
+
+
+def _create_tag_line_token(inner_pattern, inner_token, ignore_case=False):
+    # this function template-izes the tag line for a specific type of tag, which will
+    # have a different pattern and different token. otherwise, everything about a tag
+    # line is the same
+    return (
+        _create_tag_line_pattern(inner_pattern, ignore_case=ignore_case),
+        bygroups(
+            Keyword.Declaration,
+            Text.Whitespace,
+            inner_token,
+            Text.Whitespace,
+            Punctuation,
+            Text.Whitespace,
+            String,
+            Text.Whitespace,
+        ),
+    )
+
+
+class WoWTocLexer(RegexLexer):
+    """
+    Lexer for World of Warcraft TOC files.
+
+    .. versionadded:: 2.13
+    """
+
+    name = "World of Warcraft TOC"
+    aliases = ["wowtoc"]
+    filenames = ["*.toc"]
+
+    tokens = {
+        "root": [
+            # official localized tags, Notes and Title
+            # (normal part is insensitive, locale part is sensitive)
+            _create_tag_line_token(
+                r"((?:[nN][oO][tT][eE][sS]|[tT][iI][tT][lL][eE])-(?:ptBR|zhCN|enCN|frFR|deDE|itIT|esMX|ptPT|koKR|ruRU|esES|zhTW|enTW|enGB|enUS))",
+                Name.Builtin,
+            ),
+            # other official tags
+            _create_tag_line_token(
+                r"(Interface|Title|Notes|RequiredDeps|Dep[^: ]*|OptionalDeps|LoadOnDemand|LoadWith|LoadManagers|SavedVariablesPerCharacter|SavedVariables|DefaultState|Secure|Author|Version)",
+                Name.Builtin,
+                ignore_case=True,
+            ),
+            # user-defined tags
+            _create_tag_line_token(
+                r"(X-[^: ]*)",
+                Name.Variable,
+                ignore_case=True,
+            ),
+            # non-conforming tags, but still valid
+            _create_tag_line_token(
+                r"([^: ]*)",
+                Name.Other,
+            ),
+            
+            # Comments
+            (r"^#.*$", Comment),
+            
+            # Addon Files
+            (r"^.+$", Name),
+        ]
+    }
+
+    def analyse_text(text):
+        # at time of writing, this file suffix conflict's with one of Tex's in
+        # markup.py. Tex's anaylse_text() appears to be definitive (binary) and does not
+        # share any likeness to WoW TOCs, which means we wont have to compete with it by
+        # abitrary increments in score.
+        
+        result = 0
+
+        # while not required, an almost certain marker of WoW TOC's is the interface tag
+        # if this tag is omitted, players will need to opt-in to loading the addon with
+        # an options change ("Load out of date addons"). the value is also standardized:
+        # `<major><minor><patch>`, with minor and patch being two-digit zero-padded.
+        interface_pattern = _create_tag_line_pattern(r"(Interface)", ignore_case=True)
+        match = re.search(interface_pattern, text)
+        if match and re.match(r"(\d+)(\d{2})(\d{2})", match.group(7)):
+            result += 0.8
+            
+        casefolded = text.casefold()
+        # Lua file listing is good marker too, but probably conflicts with many other
+        # lexers
+        if ".lua" in casefolded:
+            result += 0.1
+        # ditto for XML files, but they're less used in WoW TOCs
+        if ".xml" in casefolded:
+            result += 0.05
+
+        return result
diff --git a/scripts/count_token_references.py b/scripts/count_token_references.py
index e235cc40..8e798c2d 100755
--- a/scripts/count_token_references.py
+++ b/scripts/count_token_references.py
@@ -72,7 +72,10 @@ def fetch_lexer_sources():
     to a list of lines.
     """
     lexer_dir = (pathlib.Path(__file__).parent / "../pygments/lexers").resolve()
-    lexer_sources = {fn: fn.read_text().splitlines(keepends=False) for fn in lexer_dir.glob("*.py")}
+    lexer_sources = {
+        fn: fn.read_text(encoding='utf-8').splitlines(keepends=False)
+        for fn in lexer_dir.glob("*.py")
+    }
     return lexer_sources
 
 
diff --git a/tests/examplefiles/wowtoc/comments.toc b/tests/examplefiles/wowtoc/comments.toc
new file mode 100644
index 00000000..381fd898
--- /dev/null
+++ b/tests/examplefiles/wowtoc/comments.toc
@@ -0,0 +1,6 @@
+#
+#a
+# a comment
+# a comment with a # in it
+## no comma, thus a comment
+## has space: and is thus, a comment
\ No newline at end of file
diff --git a/tests/examplefiles/wowtoc/comments.toc.output b/tests/examplefiles/wowtoc/comments.toc.output
new file mode 100644
index 00000000..2d69add1
--- /dev/null
+++ b/tests/examplefiles/wowtoc/comments.toc.output
@@ -0,0 +1,17 @@
+'#'           Comment
+'\n'          Text
+
+'#a'          Comment
+'\n'          Text
+
+'# a comment' Comment
+'\n'          Text
+
+'# a comment with a # in it' Comment
+'\n'          Text
+
+'## no comma, thus a comment' Comment
+'\n'          Text
+
+'## has space: and is thus, a comment' Comment
+'\n'          Text
diff --git a/tests/examplefiles/wowtoc/files.toc b/tests/examplefiles/wowtoc/files.toc
new file mode 100644
index 00000000..172cf102
--- /dev/null
+++ b/tests/examplefiles/wowtoc/files.toc
@@ -0,0 +1,3 @@
+a
+Foo.lua
+Spaces allowed.lua
\ No newline at end of file
diff --git a/tests/examplefiles/wowtoc/files.toc.output b/tests/examplefiles/wowtoc/files.toc.output
new file mode 100644
index 00000000..135f22c3
--- /dev/null
+++ b/tests/examplefiles/wowtoc/files.toc.output
@@ -0,0 +1,8 @@
+'a'           Name
+'\n'          Text
+
+'Foo.lua'     Name
+'\n'          Text
+
+'Spaces allowed.lua' Name
+'\n'          Text
diff --git a/tests/examplefiles/wowtoc/official_tags.toc b/tests/examplefiles/wowtoc/official_tags.toc
new file mode 100644
index 00000000..e8c017e5
--- /dev/null
+++ b/tests/examplefiles/wowtoc/official_tags.toc
@@ -0,0 +1,63 @@
+## TiTlE-ptBR: value
+## TiTlE-zhCN: value
+## TiTlE-enCN: value
+## TiTlE-frFR: value
+## TiTlE-deDE: value
+## TiTlE-itIT: value
+## TiTlE-esMX: value
+## TiTlE-ptPT: value
+## TiTlE-koKR: value
+## TiTlE-ruRU: value
+## TiTlE-esES: value
+## TiTlE-zhTW: value
+## TiTlE-enTW: value
+## TiTlE-enGB: value
+## TiTlE-enUS: value
+## NoTeS-ptBR: value
+## NoTeS-zhCN: value
+## NoTeS-enCN: value
+## NoTeS-frFR: value
+## NoTeS-deDE: value
+## NoTeS-itIT: value
+## NoTeS-esMX: value
+## NoTeS-ptPT: value
+## NoTeS-koKR: value
+## NoTeS-ruRU: value
+## NoTeS-esES: value
+## NoTeS-zhTW: value
+## NoTeS-enTW: value
+## NoTeS-enGB: value
+## NoTeS-enUS: value
+## Interface: value
+## interface: value
+## Title: value
+## title: value
+## Notes: value
+## notes: value
+## RequiredDeps: value
+## requireddeps: value
+## Dependencies: value
+## dependencies: value
+## OptionalDeps: value
+## optionaldeps: value
+## LoadOnDemand: value
+## loadondemand: value
+## LoadWith: value
+## loadwith: value
+## LoadManagers: value
+## loadmanagers: value
+## SavedVariablesPerCharacter: value
+## savedvariablespercharacter: value
+## SavedVariables: value
+## savedvariables: value
+## DefaultState: value
+## defaultstate: value
+## Secure: value
+## secure: value
+## Author: value
+## author: value
+## Version: value
+## version: value
+## Dep: value
+## dep: value
+## DepSomething: value
diff --git a/tests/examplefiles/wowtoc/official_tags.toc.output b/tests/examplefiles/wowtoc/official_tags.toc.output
new file mode 100644
index 00000000..90037ea0
--- /dev/null
+++ b/tests/examplefiles/wowtoc/official_tags.toc.output
@@ -0,0 +1,503 @@
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-ptBR'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-zhCN'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-enCN'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-frFR'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-deDE'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-itIT'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-esMX'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-ptPT'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-koKR'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-ruRU'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-esES'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-zhTW'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-enTW'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-enGB'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'TiTlE-enUS'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-ptBR'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-zhCN'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-enCN'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-frFR'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-deDE'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-itIT'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-esMX'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-ptPT'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-koKR'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-ruRU'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-esES'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-zhTW'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-enTW'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-enGB'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoTeS-enUS'  Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Interface'   Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'interface'   Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Title'       Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'title'       Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Notes'       Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'notes'       Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'RequiredDeps' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'requireddeps' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Dependencies' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'dependencies' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'OptionalDeps' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'optionaldeps' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'LoadOnDemand' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'loadondemand' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'LoadWith'    Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'loadwith'    Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'LoadManagers' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'loadmanagers' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'SavedVariablesPerCharacter' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'savedvariablespercharacter' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'SavedVariables' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'savedvariables' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'DefaultState' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'defaultstate' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Secure'      Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'secure'      Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Author'      Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'author'      Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Version'     Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'version'     Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Dep'         Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'dep'         Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'DepSomething' Name.Builtin
+':'           Punctuation
+' '           Text.Whitespace
+'value'       Literal.String
+'\n'          Text
diff --git a/tests/examplefiles/wowtoc/other_tags.toc b/tests/examplefiles/wowtoc/other_tags.toc
new file mode 100644
index 00000000..30ed8478
--- /dev/null
+++ b/tests/examplefiles/wowtoc/other_tags.toc
@@ -0,0 +1,6 @@
+## X-Foo: Value
+## Non-Conforming: Value
+##no:whitespace
+##      plenty   :    of   whitespace   
+## NoValueOk:
+## other	intermediate	non-space	whitespace: is ok
diff --git a/tests/examplefiles/wowtoc/other_tags.toc.output b/tests/examplefiles/wowtoc/other_tags.toc.output
new file mode 100644
index 00000000..15acbb52
--- /dev/null
+++ b/tests/examplefiles/wowtoc/other_tags.toc.output
@@ -0,0 +1,45 @@
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'X-Foo'       Name.Variable
+':'           Punctuation
+' '           Text.Whitespace
+'Value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'Non-Conforming' Name.Other
+':'           Punctuation
+' '           Text.Whitespace
+'Value'       Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+'no'          Name.Other
+':'           Punctuation
+'whitespace'  Literal.String
+'\n'          Text
+
+'##'          Keyword.Declaration
+'      '      Text.Whitespace
+'plenty'      Name.Other
+'   '         Text.Whitespace
+':'           Punctuation
+'    '        Text.Whitespace
+'of   whitespace' Literal.String
+'   '         Text.Whitespace
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'NoValueOk'   Name.Other
+':'           Punctuation
+'\n'          Text
+
+'##'          Keyword.Declaration
+' '           Text.Whitespace
+'other\tintermediate\tnon-space\twhitespace' Name.Other
+':'           Punctuation
+' '           Text.Whitespace
+'is ok'       Literal.String
+'\n'          Text