Help: Override pygments CMakeLexer to support <..> and [..]

* The code snippets in the docs consist of CMake code mixed
  with syntax definition punctuation like < > [ ] ... Therefore
  a pure CMake lexer is inadequate. Here it is replaced by a
  CMake syntax definition parser.
* Fixed syntax definition snippets in FindPkgConfig.cmake to
  make best use of syntax highlighting. This source file is the
  hardest to support because it contains comparison operators
  <= = >=, which need special attention to avoid confusion
  with the placeholder indicators <...>.
* Fixed syntax in execute_process.rst (there were unbalanced
  brackets).
* Disabled syntax highlighting for long string examples in
  cmake-language.7.rst.
* No highlighting of removed syntax in CMP0049
* To inspect the outcome of this patch, see e.g. the pages
  * manual/cmake-buildsystem.7.html
  * module/ExternalProject.html
  * module/FindPkgConfig.html
  which are particularly rich in complex code snippets.

3 files changed, 77 insertions, 6 deletions

diff --git a/Utilities/Sphinx/cmake.py b/Utilities/Sphinx/cmake.py
index ebf44da45a..882cdc1b73 100644
--- a/Utilities/Sphinx/cmake.py
+++ b/Utilities/Sphinx/cmake.py
@@ -4,14 +4,55 @@
 import os
 import re
 
-# Monkey patch for pygments reporting an error when generator expressions are
-# used.
-# https://bitbucket.org/birkenfeld/pygments-main/issue/942/cmake-generator-expressions-not-handled
+# Override much of pygments' CMakeLexer.
+# We need to parse CMake syntax definitions, not CMake code.
+
+# For hard test cases that use much of the syntax below, see
+# - module/FindPkgConfig.html (with "glib-2.0>=2.10 gtk+-2.0" and similar)
+# - module/ExternalProject.html (with http:// https:// git@; also has command options -E --build)
+# - manual/cmake-buildsystem.7.html (with nested $<..>; relative and absolute paths, "::")
+
 from pygments.lexers import CMakeLexer
-from pygments.token import Name, Operator
+from pygments.token import Name, Operator, Punctuation, String, Text, Comment, Generic, Whitespace, Number
 from pygments.lexer import bygroups
-CMakeLexer.tokens["args"].append(('(\\$<)(.+?)(>)',
-                                  bygroups(Operator, Name.Variable, Operator)))
+
+# Notes on regular expressions below:
+# - [\.\+-] are needed for string constants like gtk+-2.0
+# - Unix paths are recognized by '/'; support for Windows paths may be added if needed
+# - (\\.) allows for \-escapes (used in manual/cmake-language.7)
+# - $<..$<..$>..> nested occurence in cmake-buildsystem
+
+CMakeLexer.tokens["root"] = [
+  (r'\b(\w+)([ \t]*)(\()', bygroups(Name.Function, Text, Name.Function), '#push'),     # fctn(
+  (r'\(', Name.Function, '#push'),
+  (r'\)', Name.Function, '#pop'),
+  (r'\[', Punctuation, '#push'),
+  (r'\]', Punctuation, '#pop'),
+  (r'[|;,.=*\-]', Punctuation),
+  (r'\\\\', Punctuation),                                   # used in commands/source_group
+  (r'[:]', Operator),
+  (r'[<>]=', Punctuation),                                  # used in FindPkgConfig.cmake
+  (r'\$<', Operator, '#push'),                              # $<...>
+  (r'<[^<|]+?>(\w*\.\.\.)?', Name.Variable),                # <expr>
+  (r'(\$\w*\{)(.+?)(\})', bygroups(Operator, Name.Tag, Operator)),   # ${..} $ENV{..}
+  (r'([A-Z]+\{)(.+?)(\})', bygroups(Operator, Name.Tag, Operator)),  # DATA{ ...}
+  (r'[a-z]+(@|(://))((\\.)|[\w.+-:/\\])+', Name.Attribute),          # URL, git@, ...
+  (r'/\w[\w\.\+-/\\]*', Name.Attribute),                    # absolute path
+  (r'/', Name.Attribute),
+  (r'\w[\w\.\+-]*/[\w.+-/\\]*', Name.Attribute),            # relative path
+  (r'[A-Z]((\\.)|[\w.+-])*[a-z]((\\.)|[\w.+-])*', Name.Builtin), # initial A-Z, contains a-z
+  (r'@?[A-Z][A-Z0-9_]*', Name.Constant),
+  (r'[a-z_]((\\;)|(\\ )|[\w.+-])*', Name.Builtin),
+  (r'[0-9][0-9\.]*', Number),
+  (r'(?s)"(\\"|[^"])*"', String),                           # "string"
+  (r'\.\.\.', Name.Variable),
+  (r'<', Operator, '#push'),                                # <..|..> is different from <expr>
+  (r'>', Operator, '#pop'),
+  (r'\n', Whitespace),
+  (r'[ \t]+', Whitespace),
+  (r'#.*\n', Comment),
+  #  (r'[^<>\])\}\|$"# \t\n]+', Name.Exception),            # fallback, for debugging only
+]
 
 # Monkey patch for sphinx generating invalid content for qcollectiongenerator
 # https://bitbucket.org/birkenfeld/sphinx/issue/1435/qthelp-builder-should-htmlescape-keywords
diff --git a/Utilities/Sphinx/colors.py b/Utilities/Sphinx/colors.py
new file mode 100644
index 0000000000..f98a4831fd
--- /dev/null
+++ b/Utilities/Sphinx/colors.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+from pygments.style import Style
+from pygments.token import Name, Comment, String, Number, Operator, Whitespace
+
+class CMakeTemplateStyle(Style):
+    """
+        for more token names, see pygments/styles.default
+    """
+
+    background_color = "#f8f8f8"
+    default_style = ""
+
+    styles = {
+        Whitespace:                "#bbbbbb",
+        Comment:                   "italic #408080",
+        Operator:                  "bold #000000",
+        String:                    "#217A21",
+        Number:                    "#105030",
+        Name.Builtin:              "#400080",          # anything lowercase
+        Name.Function:             "bold #1010A0",     # function
+        Name.Variable:             "#1080B0",          # <..>
+        Name.Tag:                  "#19177C",          # ${..}
+        Name.Constant:             "#6020E0",          # uppercase only
+        Name.Entity:               "italic #70A020",   # @..@
+        Name.Attribute:            "#906060",          # paths, URLs
+        Name.Label:                "#A0A000",          # anything left over
+        Name.Exception:            "bold #FF0000",     # for debugging only
+    }
diff --git a/Utilities/Sphinx/conf.py.in b/Utilities/Sphinx/conf.py.in
index f52ccd1881..70ba08032f 100644
--- a/Utilities/Sphinx/conf.py.in
+++ b/Utilities/Sphinx/conf.py.in
@@ -15,6 +15,7 @@ project = 'CMake'
 copyright = '@conf_copyright@'
 version = '@conf_version@' # feature version
 release = '@conf_release@' # full version string
+pygments_style = 'colors.CMakeTemplateStyle'
 
 primary_domain = 'cmake'
 highlight_language = 'none'