test: change how we keep mismatched actual output

Now when a goldtest has a failure, the actual mismatched output will be written
to the tests/actual directory.

Along the way, I removed some obsolete settings which were only used by unittest
and unittest_mixins, which we no longer use:

- COVERAGE_KEEP_TMP
- COVERAGE_ENV_ID
- $TMPDIR/coverage_test

8 files changed, 22 insertions, 25 deletions

diff --git a/Makefile b/Makefile
index 4d64c079..4038229e 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,7 @@ clean: clean_platform                   ## Remove artifacts of test execution, i
 	rm -rf doc/_build doc/_spell doc/sample_html_beta
 	rm -rf tmp
 	rm -rf .cache .pytest_cache .hypothesis
-	rm -rf $$TMPDIR/coverage_test
+	rm -rf tests/actual
 	-make -C tests/gold/html clean
 
 sterile: clean                          ## Remove all non-controlled content, even if expensive.
diff --git a/doc/contributing.rst b/doc/contributing.rst
index 09e889c7..1ccb4061 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -128,10 +128,6 @@ these as 1 to use them:
 - COVERAGE_AST_DUMP: will dump the AST tree as it is being used during code
   parsing.
 
-- COVERAGE_KEEP_TMP: keeps the temporary directories in which tests are run.
-  This makes debugging tests easier. The temporary directories are at
-  ``$TMPDIR/coverage_test/*``, and are named for the test that made them.
-
 
 Of course, run all the tests on every version of Python you have, before
 submitting a change.
diff --git a/igor.py b/igor.py
index c31d21b3..9db90f89 100644
--- a/igor.py
+++ b/igor.py
@@ -124,9 +124,6 @@ def run_tests(tracer, *runner_args):
     """The actual running of tests."""
     if 'COVERAGE_TESTING' not in os.environ:
         os.environ['COVERAGE_TESTING'] = "True"
-    # $set_env.py: COVERAGE_ENV_ID - Use environment-specific test directories.
-    if 'COVERAGE_ENV_ID' in os.environ:
-        os.environ['COVERAGE_ENV_ID'] = make_env_id(tracer)
     print_banner(label_for_tracer(tracer))
 
     return pytest.main(list(runner_args))
diff --git a/tests/coveragetest.py b/tests/coveragetest.py
index 80e8853a..e5543a0b 100644
--- a/tests/coveragetest.py
+++ b/tests/coveragetest.py
@@ -51,15 +51,6 @@ class CoverageTest(
     # Let stderr go to stderr, pytest will capture it for us.
     show_stderr = True
 
-    # Temp dirs go to $TMPDIR/coverage_test/*
-    temp_dir_prefix = "coverage_test/"
-    if os.getenv('COVERAGE_ENV_ID'):                        # pragma: debugging
-        temp_dir_prefix += "{}/".format(os.getenv('COVERAGE_ENV_ID'))
-
-    # Keep the temp directories if the env says to.
-    # $set_env.py: COVERAGE_KEEP_TMP - Keep the temp directories made by tests.
-    keep_temp_dir = bool(int(os.getenv("COVERAGE_KEEP_TMP", "0")))
-
     def setup_test(self):
         super().setup_test()
 
diff --git a/tests/gold/README.rst b/tests/gold/README.rst
index aec00c71..aec1d637 100644
--- a/tests/gold/README.rst
+++ b/tests/gold/README.rst
@@ -7,16 +7,13 @@ Gold files
 These are files used in comparisons for some of the tests.  Code to support
 these comparisons is in tests/goldtest.py.
 
-If gold tests are failing, it can useful to set the COVERAGE_KEEP_TMP
-environment variable.  If set, the test working directories at
-$TMPDIR/coverage_test are kept after the tests are run, so that you can
-manually inspect the differences.
+If gold tests are failing, you may need to update the gold files by copying the
+current output of the tests into the gold files. When a test fails, the actual
+output is in the tests/actual directory. Do not commit those files to git.
 
-Do this to clean the output directories and run only the failed tests while
-keeping the output::
+You can run just the failed tests again with::
 
-    rm -rf $TMPDIR/coverage_test
-    COVERAGE_KEEP_TMP=1 tox -e py37 -- --lf
+    tox -e py39 -- -n 0 --lf
 
 The saved HTML files in the html directories can't be viewed properly without
 the supporting CSS and Javascript files. But we don't want to save copies of
diff --git a/tests/gold/html/Makefile b/tests/gold/html/Makefile
index 604ece7a..c10ede3f 100644
--- a/tests/gold/html/Makefile
+++ b/tests/gold/html/Makefile
@@ -18,6 +18,7 @@ clean:		## Remove the effects of this Makefile.
 	git clean -fq .
 
 update-gold:	## Copy output files from latest tests to gold files.
+	echo Note: this doesn't work now, it has to be updated for tests/actual
 	@for sub in $$TMPDIR/coverage_test/*HtmlGoldTests*/out; do \
 		rsync --verbose --existing --recursive $$sub/ . ; \
 	done ; \
diff --git a/tests/goldtest.py b/tests/goldtest.py
index b9d59217..96d3cf81 100644
--- a/tests/goldtest.py
+++ b/tests/goldtest.py
@@ -67,6 +67,14 @@ def compare(
     expected_only = fnmatch_list(dc.left_only, file_pattern)
     actual_only = fnmatch_list(dc.right_only, file_pattern)
 
+    def save_mismatch(f):
+        """Save a mismatched result to tests/actual."""
+        save_path = expected_dir.replace("/gold/", "/actual/")
+        os.makedirs(save_path, exist_ok=True)
+        with open(os.path.join(save_path, f), "w") as savef:
+            with open(os.path.join(actual_dir, f)) as readf:
+                savef.write(readf.read())
+
     # filecmp only compares in binary mode, but we want text mode.  So
     # look through the list of different files, and compare them
     # ourselves.
@@ -95,6 +103,12 @@ def compare(
             print(f":::: diff {expected_file!r} and {actual_file!r}")
             print("\n".join(difflib.Differ().compare(expected, actual)))
             print(f":::: end diff {expected_file!r} and {actual_file!r}")
+            save_mismatch(f)
+
+    if not actual_extra:
+        for f in actual_only:
+            save_mismatch(f)
+
     assert not text_diff, "Files differ: %s" % '\n'.join(text_diff)
 
     assert not expected_only, f"Files in {expected_dir} only: {expected_only}"
diff --git a/tests/test_html.py b/tests/test_html.py
index a0ab2d4a..94ceb902 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -634,6 +634,7 @@ def compare_html(expected, actual, extra_scrubs=None):
         (filepath_to_regex(flat_rootname(str(os.getcwd()))), '_TEST_TMPDIR'),
         (filepath_to_regex(abs_file(os.getcwd())), 'TEST_TMPDIR'),
         (filepath_to_regex(flat_rootname(str(abs_file(os.getcwd())))), '_TEST_TMPDIR'),
+        # Old format of test directories that could be in the gold files.
         (r'/private/var/folders/[\w/]{35}/coverage_test/tests_test_html_\w+_\d{8}', 'TEST_TMPDIR'),
         (r'_private_var_folders_\w{35}_coverage_test_tests_test_html_\w+_\d{8}', '_TEST_TMPDIR'),
     ]