test: test test/goldtest.pynedbat/test_goldtest

3 files changed, 108 insertions, 44 deletions

diff --git a/tests/gold/testing/gettysburg.txt b/tests/gold/testing/gettysburg.txt
new file mode 100644
index 00000000..e759ba87
--- /dev/null
+++ b/tests/gold/testing/gettysburg.txt
@@ -0,0 +1,4 @@
+Four score and seven years ago our fathers brought forth upon this continent, a
+new nation, conceived in Liberty, and dedicated to the proposition that all men
+are created equal.
+11/19/1863, Gettysburg, Pennsylvania
diff --git a/tests/goldtest.py b/tests/goldtest.py
index 57fa5c06..4f3f882a 100644
--- a/tests/goldtest.py
+++ b/tests/goldtest.py
@@ -9,10 +9,10 @@ import fnmatch
 import os
 import os.path
 import re
-import sys
 import xml.etree.ElementTree
 
 from tests.coveragetest import TESTS_DIR
+from tests.helpers import os_sep
 
 
 def gold_path(path):
@@ -20,58 +20,34 @@ def gold_path(path):
     return os.path.join(TESTS_DIR, "gold", path)
 
 
-def versioned_directory(d):
-    """Find a subdirectory of d specific to the Python version.
-    For example, on Python 3.6.4 rc 1, it returns the first of these
-    directories that exists::
-        d/3.6.4.candidate.1
-        d/3.6.4.candidate
-        d/3.6.4
-        d/3.6
-        d/3
-        d
-    Returns: a string, the path to an existing directory.
-    """
-    ver_parts = list(map(str, sys.version_info))
-    for nparts in range(len(ver_parts), -1, -1):
-        version = ".".join(ver_parts[:nparts])
-        subdir = os.path.join(d, version)
-        if os.path.exists(subdir):
-            return subdir
-    raise Exception(f"Directory missing: {d}")                  # pragma: only failure
-
-
 def compare(
         expected_dir, actual_dir, file_pattern=None,
         actual_extra=False, scrubs=None,
         ):
     """Compare files matching `file_pattern` in `expected_dir` and `actual_dir`.
 
-    A version-specific subdirectory of `expected_dir` will be used if
-    it exists.
-
     `actual_extra` true means `actual_dir` can have extra files in it
     without triggering an assertion.
 
     `scrubs` is a list of pairs: regexes to find and replace to scrub the
     files of unimportant differences.
 
-    An assertion will be raised if the directories fail one of their
-    matches.
+    If a comparison fails, a message will be written to stdout, the original
+    unscrubbed output of the test will be written to an "/actual/" directory
+    alongside the "/gold/" directory, and an assertion will be raised.
 
     """
     __tracebackhide__ = True    # pytest, please don't show me this function.
-
-    expected_dir = versioned_directory(expected_dir)
+    assert os_sep("/gold/") in expected_dir
 
     dc = filecmp.dircmp(expected_dir, actual_dir)
     diff_files = fnmatch_list(dc.diff_files, file_pattern)
     expected_only = fnmatch_list(dc.left_only, file_pattern)
     actual_only = fnmatch_list(dc.right_only, file_pattern)
 
-    def save_mismatch(f):                                   # pragma: only failure
+    def save_mismatch(f):
         """Save a mismatched result to tests/actual."""
-        save_path = expected_dir.replace("/gold/", "/actual/")
+        save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/"))
         os.makedirs(save_path, exist_ok=True)
         with open(os.path.join(save_path, f), "w") as savef:
             with open(os.path.join(actual_dir, f)) as readf:
@@ -82,7 +58,6 @@ def compare(
     # ourselves.
     text_diff = []
     for f in diff_files:
-
         expected_file = os.path.join(expected_dir, f)
         with open(expected_file) as fobj:
             expected = fobj.read()
@@ -98,13 +73,13 @@ def compare(
         if scrubs:
             expected = scrub(expected, scrubs)
             actual = scrub(actual, scrubs)
-        if expected != actual:                              # pragma: only failure
+        if expected != actual:
             text_diff.append(f'{expected_file} != {actual_file}')
             expected = expected.splitlines()
             actual = actual.splitlines()
-            print(f":::: diff {expected_file!r} and {actual_file!r}")
+            print(f":::: diff '{expected_file}' and '{actual_file}'")
             print("\n".join(difflib.Differ().compare(expected, actual)))
-            print(f":::: end diff {expected_file!r} and {actual_file!r}")
+            print(f":::: end diff '{expected_file}' and '{actual_file}'")
             save_mismatch(f)
 
     if not actual_extra:                                    # pragma: only failure
@@ -118,15 +93,6 @@ def compare(
         assert not actual_only, f"Files in {actual_dir} only: {actual_only}"
 
 
-def canonicalize_xml(xtext):
-    """Canonicalize some XML text."""
-    root = xml.etree.ElementTree.fromstring(xtext)
-    for node in root.iter():
-        node.attrib = dict(sorted(node.items()))
-    xtext = xml.etree.ElementTree.tostring(root)
-    return xtext.decode("utf-8")
-
-
 def contains(filename, *strlist):
     """Check that the file contains all of a list of strings.
 
@@ -190,6 +156,15 @@ def doesnt_contain(filename, *strlist):
 
 # Helpers
 
+def canonicalize_xml(xtext):
+    """Canonicalize some XML text."""
+    root = xml.etree.ElementTree.fromstring(xtext)
+    for node in root.iter():
+        node.attrib = dict(sorted(node.items()))
+    xtext = xml.etree.ElementTree.tostring(root)
+    return xtext.decode("utf-8")
+
+
 def fnmatch_list(files, file_pattern):
     """Filter the list of `files` to only those that match `file_pattern`.
     If `file_pattern` is None, then return the entire list of files.
diff --git a/tests/test_goldtest.py b/tests/test_goldtest.py
new file mode 100644
index 00000000..fcd41579
--- /dev/null
+++ b/tests/test_goldtest.py
@@ -0,0 +1,85 @@
+# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
+# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
+
+"""Tests of the helpers in goldtest.py"""
+
+import os.path
+import re
+
+import pytest
+
+from tests.coveragetest import CoverageTest, TESTS_DIR
+from tests.goldtest import compare, gold_path
+from tests.helpers import os_sep, re_line, remove_tree
+
+GOOD_GETTY = """\
+Four score and seven years ago our fathers brought forth upon this continent, a
+new nation, conceived in Liberty, and dedicated to the proposition that all men
+are created equal.
+11/19/9999, Gettysburg, Pennsylvania
+"""
+
+BAD_GETTY = """\
+Five score and seven years ago our fathers brought forth upon this continent, a
+new nation, conceived in Liberty, and dedicated to the proposition that all men
+are created equal.
+333/4444/55555, Gettysburg, Pennsylvania
+"""
+
+SCRUBS = [
+    # Numbers don't matter when comparing.
+    (r'\d+', 'D'),
+]
+
+ACTUAL_DIR = os.path.join(TESTS_DIR, "actual/testing")
+ACTUAL_GETTY_FILE = os.path.join(ACTUAL_DIR, "gettysburg.txt")
+
+GOLD_PATH_RE = re.escape(os_sep("/tests/gold/testing/gettysburg.txt"))
+OUT_PATH_RE = re.escape(os_sep("out/gettysburg.txt"))
+
+class CompareTest(CoverageTest):
+    """Tests of goldtest.py:compare()"""
+
+    def setup_test(self):
+        super().setup_test()
+        self.addCleanup(remove_tree, ACTUAL_DIR)
+
+    def test_compare_good(self):
+        self.make_file("out/gettysburg.txt", GOOD_GETTY)
+        compare(gold_path("testing"), "out", scrubs=SCRUBS)
+        self.assert_doesnt_exist(ACTUAL_GETTY_FILE)
+
+    def test_compare_bad(self):
+        self.make_file("out/gettysburg.txt", BAD_GETTY)
+
+        # compare() raises an assertion.
+        msg = rf"Files differ: .*{GOLD_PATH_RE} != {OUT_PATH_RE}"
+        with pytest.raises(AssertionError, match=msg):
+            compare(gold_path("testing"), "out", scrubs=SCRUBS)
+
+        # Stdout has a description of the diff.  The diff shows the scrubbed content.
+        stdout = self.stdout()
+        print(stdout)
+        assert "- Four score" in stdout
+        assert "+ Five score" in stdout
+        assert re_line(stdout, rf"^:::: diff '.*{GOLD_PATH_RE}' and '{OUT_PATH_RE}'")
+        assert re_line(stdout, rf"^:::: end diff '.*{GOLD_PATH_RE}' and '{OUT_PATH_RE}'")
+        assert "  D/D/D, Gettysburg, Pennsylvania" in stdout
+
+        # The actual file was saved.
+        with open(ACTUAL_GETTY_FILE) as f:
+            saved = f.read()
+        assert saved == BAD_GETTY
+
+    def test_compare_good_needs_scrubs(self):
+        # Comparing the "good" result without scrubbing the variable parts will fail.
+        self.make_file("out/gettysburg.txt", GOOD_GETTY)
+
+        # compare() raises an assertion.
+        msg = rf"Files differ: .*{GOLD_PATH_RE} != {OUT_PATH_RE}"
+        with pytest.raises(AssertionError, match=msg):
+            compare(gold_path("testing"), "out")
+
+        stdout = self.stdout()
+        assert "- 11/19/1863, Gettysburg, Pennsylvania" in stdout
+        assert "+ 11/19/9999, Gettysburg, Pennsylvania" in stdout