Add another check script for whitespace.

Add a script which checks for whitespace tokens, similar to the script checking for
repeated tokens. Also move some functionality shared between them into a utility file,
and make check_repeated_token PEP8 compliant.

1 files changed, 40 insertions, 0 deletions

diff --git a/scripts/check_whitespace_token.py b/scripts/check_whitespace_token.py
new file mode 100644
index 00000000..9fb56ab3
--- /dev/null
+++ b/scripts/check_whitespace_token.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+    Checker for whitespace tokens
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Helper script to find whitespace which is not of token type `Whitespace`
+
+    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import argparse
+import sys
+import re
+
+from utility import unpack_output_file, process_output_files
+
+
+def check_file(path):
+    whitespace_re = re.compile('\s+')
+
+    for value, token, linenumber in unpack_output_file(path):
+        if whitespace_re.fullmatch(value) and 'Whitespace' not in token:
+            print(f'{path}:{linenumber}')
+            return False
+
+    return True
+
+
+def main(args):
+    if process_output_files(args.TEST_ROOT, check_file) > 0:
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('TEST_ROOT',
+                        help='Root directory containing the tests')
+    args = parser.parse_args()
+    sys.exit(main(args))