From 019f0996ec3dcfd6c8237b336ac8d81256e58239 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 21 Mar 2023 08:36:25 -0400 Subject: test: provide more guidance about working with gold files And make it easier to add new gold tests. --- tests/gold/README.rst | 18 ++++++++++++++---- tests/gold/html/Makefile | 2 +- tests/goldtest.py | 6 +++++- tests/test_goldtest.py | 6 +++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/gold/README.rst b/tests/gold/README.rst index aec1d637..ec408182 100644 --- a/tests/gold/README.rst +++ b/tests/gold/README.rst @@ -9,16 +9,26 @@ these comparisons is in tests/goldtest.py. If gold tests are failing, you may need to update the gold files by copying the current output of the tests into the gold files. When a test fails, the actual -output is in the tests/actual directory. Do not commit those files to git. +output is in the tests/actual directory. Those files are ignored by git. -You can run just the failed tests again with:: +There's a Makefile in the html directory for working with gold files and their +associated support files. If actual outputs have changed so that comparisons +are failing, but the new output is correct, you can use "make update-gold" to +copy the actual output as the new gold files. + +If you have added a gold test, you'll need to manually copy the tests/actual +files to tests/gold. + +Once you've copied the actual results to the gold files, or to check your work +again, you can run just the failed tests again with:: tox -e py39 -- -n 0 --lf The saved HTML files in the html directories can't be viewed properly without the supporting CSS and Javascript files. But we don't want to save copies of -those files in every subdirectory. There's a Makefile in the html directory -for working with the saved copies of the support files. +those files in every subdirectory. The make target "make complete" in +tests/gold/html will copy the support file so you can open the HTML files to +see how they look. If the output files are correct, you can update the gold files with "make update-gold". If there are version-specific gold files (for example, diff --git a/tests/gold/html/Makefile b/tests/gold/html/Makefile index 7be71f84..1b75f73d 100644 --- a/tests/gold/html/Makefile +++ b/tests/gold/html/Makefile @@ -17,7 +17,7 @@ complete: ## Copy support files into directories so the HTML can be viewed prop clean: ## Remove the effects of this Makefile. @git clean -fq . -update-gold: ## Copy output files from latest tests to gold files. +update-gold: ## Copy actual output files from latest tests to gold files. @for sub in ../../actual/html/*; do \ rsync --verbose --existing --recursive $$sub/ $$(basename $$sub) ; \ done ; \ diff --git a/tests/goldtest.py b/tests/goldtest.py index 12a04af6..f7bd2e9b 100644 --- a/tests/goldtest.py +++ b/tests/goldtest.py @@ -46,6 +46,8 @@ def compare( """ __tracebackhide__ = True # pytest, please don't show me this function. assert os_sep("/gold/") in expected_dir + assert os.path.exists(actual_dir) + os.makedirs(expected_dir, exist_ok=True) dc = filecmp.dircmp(expected_dir, actual_dir) diff_files = _fnmatch_list(dc.diff_files, file_pattern) @@ -56,9 +58,11 @@ def compare( """Save a mismatched result to tests/actual.""" save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/")) os.makedirs(save_path, exist_ok=True) - with open(os.path.join(save_path, f), "w") as savef: + save_file = os.path.join(save_path, f) + with open(save_file, "w") as savef: with open(os.path.join(actual_dir, f)) as readf: savef.write(readf.read()) + print(os_sep(f"Saved actual output to '{save_file}': see tests/gold/README.rst")) # filecmp only compares in binary mode, but we want text mode. So # look through the list of different files, and compare them diff --git a/tests/test_goldtest.py b/tests/test_goldtest.py index def5ee90..297f21b0 100644 --- a/tests/test_goldtest.py +++ b/tests/test_goldtest.py @@ -13,7 +13,7 @@ import pytest from tests.coveragetest import CoverageTest, TESTS_DIR from tests.goldtest import compare, gold_path from tests.goldtest import contains, contains_any, contains_rx, doesnt_contain -from tests.helpers import re_line, remove_tree +from tests.helpers import os_sep, re_line, remove_tree GOOD_GETTY = """\ Four score and seven years ago our fathers brought forth upon this continent, a @@ -73,6 +73,10 @@ class CompareTest(CoverageTest): assert "+ Five score" in stdout assert re_line(rf"^:::: diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) assert re_line(rf"^:::: end diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) + assert ( + os_sep(f"Saved actual output to '{ACTUAL_GETTY_FILE}': see tests/gold/README.rst") + in os_sep(stdout) + ) assert " D/D/D, Gxxx, Pennsylvania" in stdout # The actual file was saved. -- cgit v1.2.1