Merge mapping file generation scripts (#2152)

Use a unified script, to reduce code duplication and in preparation
for doing a similar thing with styles and filters.  The new script
also uses a bit more modern Python APIs (e.g., pathlib).

Unlike the previous scripts, it does not replace replace CRLF with LF
because Git should do that itself.

1 files changed, 53 insertions, 0 deletions

diff --git a/scripts/gen_mapfiles.py b/scripts/gen_mapfiles.py
new file mode 100644
index 00000000..5fd084ad
--- /dev/null
+++ b/scripts/gen_mapfiles.py
@@ -0,0 +1,53 @@
+"""
+    scripts/gen_mapfiles.py
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Regenerate mapping files.
+
+    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from importlib import import_module
+from pathlib import Path
+import re
+import sys
+
+top_src_dir = Path(__file__).parent.parent
+pygments_package = top_src_dir / 'pygments'
+sys.path.insert(0, pygments_package.parent.resolve())
+
+from pygments.util import docstring_headline
+
+def main():
+    for key in ['lexers', 'formatters']:
+        lines = []
+        for file in (pygments_package / key).glob('[!_]*.py'):
+            module_name = '.'.join(file.relative_to(pygments_package.parent).with_suffix('').parts)
+            print(module_name)
+            module = import_module(module_name)
+            for obj_name in module.__all__:
+                obj = getattr(module, obj_name)
+                desc = (module_name, obj.name, tuple(obj.aliases), tuple(obj.filenames))
+                if key == 'lexers':
+                    desc += (tuple(obj.mimetypes),)
+                elif key == 'formatters':
+                    desc += (docstring_headline(obj),)
+                else:
+                    assert False
+                lines.append(f'    {obj_name!r}: {desc!r},')
+        # Sort to make diffs minimal.
+        lines.sort()
+        new_dict = '\n'.join(lines)
+        content = f'''# Automatically generated by scripts/gen_mapfiles.py.
+# DO NOT EDIT BY HAND; run `make mapfiles` instead.
+
+{key.upper()} = {{
+{new_dict}
+}}
+'''
+        (pygments_package / key / '_mapping.py').write_text(content, encoding='utf8')
+        print(f'=== {len(lines)} {key} processed.')
+
+if __name__ == '__main__':
+    main()