Replace Makefile with tox (#2331)

Porting notes:

- tox handles Python environments automatically. Remove a bit of PYTHONPATH
  manipulation (that was using Python 2 code which always failed!)

- No `clean` target: `git clean -xdf` should fit the bill.

- No `reindent` target: the `reindent.py` script it was using does not
  exist (anymore?).

- No equivalent of tox-test-coverage, which was an artifact of the past,
  using nose.  Instead, the test-coverage target only is ported, which
  uses pytest, and works.

15 files changed, 131 insertions, 132 deletions

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index f2865b78..3f5c26ad 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -23,12 +23,10 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
-    - name: Install package
-      run: |
-        pip install -r requirements.txt
-        pip install .
+    - name: Install tox
+      run: pip install -r requirements.txt
     - name: Test package
-      run: pytest -W error
+      run: tox -- -W error
 
   check:
     runs-on: ubuntu-latest
@@ -37,10 +35,10 @@ jobs:
     - uses: actions/setup-python@v4
       with:
         python-version: "3.x"
-    - name: Run make check
-      run: make check
-    - name: Fail if the basic checks failed
-      run: make check
+    - name: Install tox
+      run: pip install -r requirements.txt
+    - name: Perform basic checks
+      run: tox -e check
       if: runner.os == 'Linux'
 
   check-mapfiles:
@@ -50,12 +48,14 @@ jobs:
     - uses: actions/setup-python@v4
       with:
         python-version: "3.x"
+    - name: Install tox
+      run: pip install -r requirements.txt
     - name: Regenerate mapfiles
-      run: make mapfiles
+      run: tox -e mapfiles
     - name: Fail if mapfiles changed
       run: |
         if git ls-files -m | grep mapping; then
-          echo 'Please run "make mapfiles" and add the changes to a commit.'
+          echo 'Please run "tox -e mapfiles" and add the changes to a commit.'
           exit 1
         fi
 
@@ -66,7 +66,7 @@ jobs:
     - uses: actions/setup-python@v4
       with:
         python-version: "3.x"
-    - name: Check out regexlint
-      run: git clone https://github.com/pygments/regexlint
+    - name: Install tox
+      run: pip install -r requirements.txt
     - name: Run regexlint
-      run: make regexlint REGEXLINT=`pwd`/regexlint
+      run: tox -e regexlint
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 672beaf3..d1000e04 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -5,6 +5,7 @@ on:
     branches:
       - master
 
+
 env:
   FORCE_COLOR: 1
 
@@ -22,17 +23,14 @@ jobs:
         python-version: "3.x"
     - name: Checkout Pygments
       uses: actions/checkout@v3
-    - name: Install Sphinx & WCAG contrast ratio
-      run: pip install Sphinx wcag-contrast-ratio
-    - name: Create Pyodide WASM package
-      run: cd doc && make pyodide
+    - name: Install tox
+      run: pip install -r requirements.txt
     - name: Sphinx build
       run: |
-        cd doc
-        WEBSITE_BUILD=1 make dirhtml
-        touch _build/dirhtml/.nojekyll
-        echo -e 'pygments.org\nwww.pygments.org' > _build/dirhtml/CNAME
-        echo 'Automated deployment of docs for GitHub pages.' > _build/dirhtml/README
+        tox -e web-doc -- dirhtml
+        touch doc/_build/dirhtml/.nojekyll
+        echo -e 'pygments.org\nwww.pygments.org' > doc/_build/dirhtml/CNAME
+        echo 'Automated deployment of docs for GitHub pages.' > doc/_build/dirhtml/README
     - name: Deploy to repo
       if: github.repository_owner == 'pygments'
       uses: peaceiris/actions-gh-pages@v3
diff --git a/CHANGES b/CHANGES
index b76c9e2c..f4ace2fe 100644
--- a/CHANGES
+++ b/CHANGES
@@ -35,6 +35,10 @@ Version 2.15.0
 - Move project metadata to ``pyproject.toml``, remove ``setup.py``
   and ``setup.cfg`` (#2342)
 
+- The top-level ``Makefile`` has been removed. Instead, all shortcuts
+  for developing are now defined and run through tox.  The ``doc`` folder
+  still contains a ``Makefile`` as an alternative to ``tox -e doc``.
+
 Version 2.14.0
 --------------
 (released January 1st, 2023)
diff --git a/Contributing.md b/Contributing.md
index 93da428c..e3fdedb1 100644
--- a/Contributing.md
+++ b/Contributing.md
@@ -156,8 +156,8 @@ Contribution checklist
 
     To add a new test, create a file with just your code snippet under a
     subdirectory based on your lexer's main alias. Then run
-    ``pytest --update-goldens <filename.txt>`` to auto-populate the currently
-    expected tokens. Check that they look good and check in the file.
+    ``tox -- --update-goldens <filename.txt>`` to auto-populate the
+    currently expected tokens. Check that they look good and check in the file.
 
     Also run the same command whenever you need to update the test if the
     actual produced tokens change (assuming the change is expected).
diff --git a/Makefile b/Makefile
deleted file mode 100644
index e674732c..00000000
--- a/Makefile
+++ /dev/null
@@ -1,68 +0,0 @@
-#
-# Makefile for Pygments
-# ~~~~~~~~~~~~~~~~~~~~~
-#
-# Combines scripts for common tasks.
-#
-# :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
-# :license: BSD, see LICENSE for details.
-#
-
-PYTHON ?= python3
-
-export PYTHONPATH = $(shell echo "$$PYTHONPATH"):$(shell python -c 'import os; print ":".join(os.path.abspath(line.strip()) for line in file("PYTHONPATH"))' 2>/dev/null)
-
-.PHONY: all check clean clean-pyc docs mapfiles \
-	pylint reindent test test-coverage \
-	tox-test tox-test-coverage regexlint
-
-all: clean-pyc check test
-
-check:
-	@$(PYTHON) scripts/check_crlf.py pygments build external
-	@$(PYTHON) scripts/detect_missing_analyse_text.py --skip-no-aliases
-	@pyflakes pygments | grep -v 'but unused' || true
-	@$(PYTHON) scripts/check_sources.py -i build -i dist -i pygments/lexers/_mapping.py \
-		   -i docs/build -i pygments/formatters/_mapping.py -i pygments/unistring.py \
-		   -i tests/support/empty.py
-	@$(PYTHON) scripts/count_token_references.py --minfiles=1 --maxfiles=1 \
-		   --minlines=1 --maxlines=3 --subtoken
-
-clean: clean-pyc
-	-rm -rf doc/_build build Pygments.egg-info
-	-rm -f codetags.html
-
-clean-pyc:
-	find . -name '__pycache__' -exec rm -rf {} +
-
-docs:
-	make -C doc html
-
-mapfiles:
-	$(PYTHON) scripts/gen_mapfiles.py
-
-pylint:
-	@pylint --rcfile scripts/pylintrc pygments
-
-reindent:
-	@$(PYTHON) scripts/reindent.py -r -B .
-
-TEST = tests
-
-test:
-	@$(PYTHON) -m pytest $(TEST)
-
-test-coverage:
-	@$(PYTHON) -m pytest --cov --cov-report=html --cov-report=term $(TEST)
-
-tox-test:
-	@tox -- $(TEST)
-
-tox-test-coverage:
-	@tox -- --with-coverage --cover-package=pygments --cover-erase $(TEST)
-
-RLMODULES = pygments.lexers
-
-regexlint:
-	@if [ -z "$(REGEXLINT)" ]; then echo "Please set REGEXLINT=checkout path"; exit 1; fi
-	PYTHONPATH=`pwd`:$(REGEXLINT) $(PYTHON) $(REGEXLINT)/regexlint/cmdline.py $(RLMODULES)
diff --git a/README.rst b/README.rst
index 204e46b1..d473fbca 100644
--- a/README.rst
+++ b/README.rst
@@ -16,13 +16,13 @@ Documentation
 
 ... can be found online at https://pygments.org/ or created with Sphinx by ::
 
-   make docs
+   tox -e doc
 
 By default, the documentation does not include the demo page, as it requires
 having Docker installed for building Pyodide. To build the documentation with
 the demo page, use ::
 
-   WEBSITE_BUILD=1 make docs
+   tox -e web-doc
 
 The initial build might take some time, but subsequent ones should be instant
 because of Docker caching.
@@ -59,12 +59,12 @@ significant amounts of memory. This can subsequently be used to perform a
 remote denial-of-service attack on the server if the processes are not
 terminated quickly.
 
-Unfortunately, it's practically impossible to harden Pygments itself against 
-those issues: Some regular expressions can result in "catastrophic 
+Unfortunately, it's practically impossible to harden Pygments itself against
+those issues: Some regular expressions can result in "catastrophic
 backtracking", but other bugs like incorrect matchers can also
 cause similar problems, and there is no way to find them in an automated fashion
-(short of solving the halting problem.) Pygments has extensive unit tests, 
-automated randomized testing, and is also tested by `OSS-Fuzz <https://github.com/google/oss-fuzz/tree/master/projects/pygments>`_, 
+(short of solving the halting problem.) Pygments has extensive unit tests,
+automated randomized testing, and is also tested by `OSS-Fuzz <https://github.com/google/oss-fuzz/tree/master/projects/pygments>`_,
 but we will never be able to eliminate all bugs in this area.
 
 Our recommendations are:
diff --git a/doc/Makefile b/doc/Makefile
index a0dcaaa4..f648bfa8 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -18,7 +18,6 @@ I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  pyodide    to make Pyodide with currently checked out Pygments"
 	@echo "  html       to make standalone HTML files"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
 	@echo "  singlehtml to make a single large HTML file"
@@ -42,21 +41,12 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-pyodide:
-	$(if $(test ! -f docker), $(error "Could not find Docker. Please install that before continuing."))
-	# Enable the BuildKit backend to use the --output option.
-	DOCKER_BUILDKIT=1 docker build --file pyodide/Dockerfile --output $(BUILDDIR)/pyodide/pyodide ..
-	@echo
-	@echo "Pyodide build finished. The Pyodide artifacts are in $(BUILDDIR)/pyodide."
-
 html:
-	$(if $(WEBSITE_BUILD), $(MAKE) pyodide)
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
 dirhtml:
-	$(if $(WEBSITE_BUILD), $(MAKE) pyodide)
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
diff --git a/doc/docs/lexerdevelopment.rst b/doc/docs/lexerdevelopment.rst
index 0809e75e..29bdd6ca 100644
--- a/doc/docs/lexerdevelopment.rst
+++ b/doc/docs/lexerdevelopment.rst
@@ -171,17 +171,17 @@ Add the name of your lexer class to this list (or create the list if your lexer
 is the only class in the module).
 
 Finally the lexer can be made publicly known by rebuilding the lexer mapping.
-In the root directory of the source (where the ``Makefile`` is located), run:
+In the root directory of the source (where the ``tox.ini`` file is located), run:
 
 .. code-block:: console
 
-    $ make mapfiles
+    $ tox -e mapfiles
 
 To test the new lexer, store an example file in
 ``tests/examplefiles/<alias>``.  For example, to test your
 ``DiffLexer``, add a ``tests/examplefiles/diff/example.diff`` containing a
 sample diff output.  To (re)generate the lexer output which the file is checked
-against, use the command ``pytest tests/examplefiles/diff --update-goldens``.
+against, use the command ``tox -- tests/examplefiles/diff --update-goldens``.
 
 Now you can use ``python -m pygments`` from the current root of the checkout to
 render your example to HTML:
@@ -201,7 +201,7 @@ Once the example renders as expected, you should run the complete test suite:
 
 .. code-block:: console
 
-    $ make test
+    $ tox
 
 It also tests that your lexer fulfills the lexer API and certain invariants,
 such as that the concatenation of all token text is the same as the input text.
diff --git a/doc/styles.rst b/doc/styles.rst
index a1bb0195..1c58748f 100644
--- a/doc/styles.rst
+++ b/doc/styles.rst
@@ -2,4 +2,5 @@
 
 This file is overridden by _templates/styles.html and just exists to allow the
 Styles gallery to be reliably linked from the documentation
-(since its location varies between `make html` and `make dirhtml`).
+(since its location varies between the ``html`` and ``dirhtml``
+builders).
diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py
index 6e34f960..c1890cd1 100755
--- a/pygments/formatters/_mapping.py
+++ b/pygments/formatters/_mapping.py
@@ -1,5 +1,5 @@
 # Automatically generated by scripts/gen_mapfiles.py.
-# DO NOT EDIT BY HAND; run `make mapfiles` instead.
+# DO NOT EDIT BY HAND; run `tox -e mapfiles` instead.
 
 FORMATTERS = {
     'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 6608f30f..bf56bb8e 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -1,5 +1,5 @@
 # Automatically generated by scripts/gen_mapfiles.py.
-# DO NOT EDIT BY HAND; run `make mapfiles` instead.
+# DO NOT EDIT BY HAND; run `tox -e mapfiles` instead.
 
 LEXERS = {
     'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap', '*.ABAP'), ('text/x-abap',)),
diff --git a/requirements.txt b/requirements.txt
index 05a0b12b..2bcf4dfd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,2 @@
-pytest-cov
-pytest-randomly
-pytest>=7.0
-pyflakes
-pylint
-tox
-wcag-contrast-ratio
+tox ~= 4.4
+# Other requirements are installed by tox.
diff --git a/scripts/gen_mapfiles.py b/scripts/gen_mapfiles.py
index 0b2f2e9c..bb434e11 100644
--- a/scripts/gen_mapfiles.py
+++ b/scripts/gen_mapfiles.py
@@ -39,7 +39,7 @@ def main():
         lines.sort()
         new_dict = '\n'.join(lines)
         content = f'''# Automatically generated by scripts/gen_mapfiles.py.
-# DO NOT EDIT BY HAND; run `make mapfiles` instead.
+# DO NOT EDIT BY HAND; run `tox -e mapfiles` instead.
 
 {key.upper()} = {{
 {new_dict}
diff --git a/scripts/release-checklist b/scripts/release-checklist
index 539933f7..462250f1 100644
--- a/scripts/release-checklist
+++ b/scripts/release-checklist
@@ -2,17 +2,18 @@ Release checklist
 =================
 
 * Check ``git status``
-* ``make check``
-* LATER when configured properly: ``make pylint``
 * ``tox``
+* ``tox -e check``
+* LATER when configured properly: ``tox -e pylint``
 * Update version in ``pygments/__init__.py``
 * Check pyproject.toml metadata: long description, trove classifiers
 * Update release date/code name in ``CHANGES``
 * ``git commit``
 * Wait for the CI to finish
-* ``make clean``
+* ``git clean -xdf`` (warning: removes all untracked and ignored files,
+   do a dry run with ``git clean -xdfn`` first)
 * ``python3 -m build``
-* Check the size of the generated packages. If they're significantly different from the last release, check if the repository is in a modified state and that ``make clean`` was run.
+* Check the size of the generated packages. If they're significantly different from the last release, check if the repository is in a modified state and that ``git clean`` was run.
 * ``twine upload dist/Pygments-$NEWVER*``
 * Check PyPI release page for obvious errors (like different file sizes!)
 * ``git tag -a``
diff --git a/tox.ini b/tox.ini
index 0de97559..1dcc73a7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,15 +1,93 @@
 [tox]
-envlist = py{37, 38, 39, 310, 311, 312}, lint
+envlist = py
 
 [testenv]
+description =
+  run tests with pytest (you can pass extra arguments for pytest,
+  e.g., "tox -- --update-goldens")
 deps =
-    pytest
+    pytest >= 7.0
     pytest-cov
+    pytest-randomly
     wcag-contrast-ratio
 commands = pytest {posargs}
+use_develop = True
 
-
-[testenv:lint]
+[testenv:regexlint]
+description =
+  lint regular expressions with regexlint
 deps =
     git+https://github.com/pygments/regexlint.git@master
 commands = regexlint pygments.lexers
+
+
+[testenv:pylint]
+description =
+  lint code with pylint
+deps =
+  pylint
+skip_install = True # doesn't need installing Pygments into the venv
+commands =
+  pylint --rcfile scripts/pylintrc pygments
+
+
+[testenv:check]
+description =
+  miscellaneous checks on the source code, including pyflakes
+deps =
+  flake8
+commands =
+  python scripts/check_crlf.py pygments external
+  python scripts/detect_missing_analyse_text.py --skip-no-aliases
+  # We only use pyflakes, not pycodestyle, but use it through flake8 nevertheless
+  # to be able to use the --ignore option.
+  flake8 --select F --ignore F401 pygments
+  python scripts/check_sources.py -i pygments/lexers/_mapping.py \
+        -i docs/_build -i pygments/formatters/_mapping.py -i pygments/unistring.py \
+        -i tests/support/empty.py
+  python scripts/count_token_references.py --minfiles=1 --maxfiles=1 \
+                                           --minlines=1 --maxlines=3 --subtoken
+
+[testenv:mapfiles]
+description =
+  regenerate map files
+deps =
+commands =
+  python scripts/gen_mapfiles.py
+
+
+[testenv:coverage]
+description =
+  run tests, and generate a coverage report in htmlcov/
+commands =
+  pytest --cov --cov-report=html --cov-report=term {posargs}
+
+
+[testenv:doc]
+description =
+  compile documentation with Sphinx. You can pass a builder name,
+  like "tox -e doc -- latex". You can also add extra options, like
+  "SPHINXOPTS='-D latex_paper_size=letter' tox -e doc -- latex".
+change_dir = doc
+deps =
+  sphinx
+  wcag-contrast-ratio
+commands =
+  sphinx-build -b {posargs:html} {env:SPHINXOPTS} . _build/{posargs:html}
+
+
+[testenv:web-doc]
+description =
+  same as doc, but also build the demo by compiling Pygments to WASM.
+change_dir = doc
+deps = {[testenv:doc]deps}
+allowlist_externals =
+  docker
+setenv =
+  # Enable the BuildKit backend to use the --output option.
+  DOCKER_BUILDKIT = 1
+  # Build the demo page.
+  WEBSITE_BUILD = 1
+commands =
+  docker build --file pyodide/Dockerfile --output _build/pyodide/pyodide ..
+  sphinx-build -b {posargs:html} {env:SPHINXOPTS} . _build/{posargs:html}