1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
"""A test base class for tests based on gold file comparison."""
from __future__ import annotations
import difflib
import filecmp
import fnmatch
import os
import os.path
import re
import xml.etree.ElementTree
from typing import Iterable, List, Optional, Tuple
from tests.coveragetest import TESTS_DIR
from tests.helpers import os_sep
def gold_path(path: str) -> str:
"""Get a path to a gold file for comparison."""
return os.path.join(TESTS_DIR, "gold", path)
def compare(
expected_dir: str,
actual_dir: str,
file_pattern: Optional[str] = None,
actual_extra: bool = False,
scrubs: Optional[List[Tuple[str, str]]] = None,
) -> None:
"""Compare files matching `file_pattern` in `expected_dir` and `actual_dir`.
`actual_extra` true means `actual_dir` can have extra files in it
without triggering an assertion.
`scrubs` is a list of pairs: regexes to find and replace to scrub the
files of unimportant differences.
If a comparison fails, a message will be written to stdout, the original
unscrubbed output of the test will be written to an "/actual/" directory
alongside the "/gold/" directory, and an assertion will be raised.
"""
__tracebackhide__ = True # pytest, please don't show me this function.
assert os_sep("/gold/") in expected_dir
dc = filecmp.dircmp(expected_dir, actual_dir)
diff_files = _fnmatch_list(dc.diff_files, file_pattern)
expected_only = _fnmatch_list(dc.left_only, file_pattern)
actual_only = _fnmatch_list(dc.right_only, file_pattern)
def save_mismatch(f: str) -> None:
"""Save a mismatched result to tests/actual."""
save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/"))
os.makedirs(save_path, exist_ok=True)
with open(os.path.join(save_path, f), "w") as savef:
with open(os.path.join(actual_dir, f)) as readf:
savef.write(readf.read())
# filecmp only compares in binary mode, but we want text mode. So
# look through the list of different files, and compare them
# ourselves.
text_diff = []
for f in diff_files:
expected_file = os.path.join(expected_dir, f)
with open(expected_file) as fobj:
expected = fobj.read()
if expected_file.endswith(".xml"):
expected = canonicalize_xml(expected)
actual_file = os.path.join(actual_dir, f)
with open(actual_file) as fobj:
actual = fobj.read()
if actual_file.endswith(".xml"):
actual = canonicalize_xml(actual)
if scrubs:
expected = scrub(expected, scrubs)
actual = scrub(actual, scrubs)
if expected != actual:
text_diff.append(f'{expected_file} != {actual_file}')
expected_lines = expected.splitlines()
actual_lines = actual.splitlines()
print(f":::: diff '{expected_file}' and '{actual_file}'")
print("\n".join(difflib.Differ().compare(expected_lines, actual_lines)))
print(f":::: end diff '{expected_file}' and '{actual_file}'")
save_mismatch(f)
if not actual_extra:
for f in actual_only:
save_mismatch(f)
assert not text_diff, "Files differ: " + "\n".join(text_diff)
assert not expected_only, f"Files in {expected_dir} only: {expected_only}"
if not actual_extra:
assert not actual_only, f"Files in {actual_dir} only: {actual_only}"
def contains(filename: str, *strlist: str) -> None:
"""Check that the file contains all of a list of strings.
An assert will be raised if one of the arguments in `strlist` is
missing in `filename`.
"""
__tracebackhide__ = True # pytest, please don't show me this function.
with open(filename) as fobj:
text = fobj.read()
for s in strlist:
assert s in text, f"Missing content in {filename}: {s!r}"
def contains_rx(filename: str, *rxlist: str) -> None:
"""Check that the file has lines that re.search all of the regexes.
An assert will be raised if one of the regexes in `rxlist` doesn't match
any lines in `filename`.
"""
__tracebackhide__ = True # pytest, please don't show me this function.
with open(filename) as fobj:
lines = fobj.readlines()
for rx in rxlist:
assert any(re.search(rx, line) for line in lines), (
f"Missing regex in {filename}: r{rx!r}"
)
def contains_any(filename: str, *strlist: str) -> None:
"""Check that the file contains at least one of a list of strings.
An assert will be raised if none of the arguments in `strlist` is in
`filename`.
"""
__tracebackhide__ = True # pytest, please don't show me this function.
with open(filename) as fobj:
text = fobj.read()
for s in strlist:
if s in text:
return
assert False, f"Missing content in {filename}: {strlist[0]!r} [1 of {len(strlist)}]"
def doesnt_contain(filename: str, *strlist: str) -> None:
"""Check that the file contains none of a list of strings.
An assert will be raised if any of the strings in `strlist` appears in
`filename`.
"""
__tracebackhide__ = True # pytest, please don't show me this function.
with open(filename) as fobj:
text = fobj.read()
for s in strlist:
assert s not in text, f"Forbidden content in {filename}: {s!r}"
# Helpers
def canonicalize_xml(xtext: str) -> str:
"""Canonicalize some XML text."""
root = xml.etree.ElementTree.fromstring(xtext)
for node in root.iter():
node.attrib = dict(sorted(node.items()))
return xml.etree.ElementTree.tostring(root).decode("utf-8")
def _fnmatch_list(files: List[str], file_pattern: Optional[str]) -> List[str]:
"""Filter the list of `files` to only those that match `file_pattern`.
If `file_pattern` is None, then return the entire list of files.
Returns a list of the filtered files.
"""
if file_pattern:
files = [f for f in files if fnmatch.fnmatch(f, file_pattern)]
return files
def scrub(strdata: str, scrubs: Iterable[Tuple[str, str]]) -> str:
"""Scrub uninteresting data from the payload in `strdata`.
`scrubs` is a list of (find, replace) pairs of regexes that are used on
`strdata`. A string is returned.
"""
for rx_find, rx_replace in scrubs:
strdata = re.sub(rx_find, rx_replace, strdata)
return strdata
|
