1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
#!/usr/bin/env python3
import sys
from pathlib import Path
import csv
import unicodedata
import argparse
from operator import methodcaller
def parse_codepoint(s: str, base=16):
return chr(int(s, base))
def parse_bool(s: str):
match(s):
case "True":
return True
case "False":
return False
case _:
raise ValueError(s)
def general_category(c: str):
match(unicodedata.category(c)):
case "Lu":
return "UppercaseLetter"
case "Ll":
return "LowercaseLetter"
case "Lt":
return "TitlecaseLetter"
case "Lm":
return "ModifierLetter"
case "Lo":
return "OtherLetter"
case "Mn":
return "NonSpacingMark"
case "Mc":
return "SpacingCombiningMark"
case "Me":
return "EnclosingMark"
case "Nd":
return "DecimalNumber"
case "Nl":
return "LetterNumber"
case "No":
return "OtherNumber"
case "Pc":
return "ConnectorPunctuation"
case "Pd":
return "DashPunctuation"
case "Ps":
return "OpenPunctuation"
case "Pe":
return "ClosePunctuation"
case "Pi":
return "InitialQuote"
case "Pf":
return "FinalQuote"
case "Po":
return "OtherPunctuation"
case "Sm":
return "MathSymbol"
case "Sc":
return "CurrencySymbol"
case "Sk":
return "ModifierSymbol"
case "So":
return "OtherSymbol"
case "Zs":
return "Space"
case "Zl":
return "LineSeparator"
case "Zp":
return "ParagraphSeparator"
case "Cc":
return "Control"
case "Cf":
return "Format"
case "Cs":
return "Surrogate"
case "Co":
return "PrivateUse"
case "Cn":
return "NotAssigned"
case cat:
raise ValueError((c, cat))
def check(r: bool, info):
if r:
return True
else:
raise AssertionError(info)
def make_parser():
parser = argparse.ArgumentParser()
parser.add_argument(
"csv", type=Path,
metavar="FILE",
help="CSV file to analyse."
)
parser.add_argument(
"-v", "--verbosity",
action="count", default=0,
help="Verbosity level. Default: %(default)s"
)
return parser
def check_case_mapping(case_mapping: str, char: str, raw_expected: str, verbosity: int):
got = methodcaller(case_mapping)(char)
expected = parse_codepoint(raw_expected)
if len(got) == 1:
check(got == expected, (char, got, expected))
elif verbosity > 0:
print(
f"[INFO] Skipped {case_mapping} check for: {codepoint}; "
f"“{got}” ({len(got)} chars) cannot be compared to "
f"“{expected}” (1 char)."
)
def check_case_predicate(case_predicate: str, char: str, raw_expected: str, verbosity: int):
got = methodcaller(case_predicate)(char)
expected = parse_bool(raw_expected)
check(got == expected, (char, got, expected))
if __name__ == "__main__":
parser = make_parser()
args = parser.parse_args()
path = args.csv
verbosity = args.verbosity
with path.open("rt", encoding="utf-8") as fp:
version = next(fp).strip()
if version != unicodedata.unidata_version:
sys.exit(
f"Incompatible Python Unicode version: expecting “{version}”, "
f"but got: “{unicodedata.unidata_version}”."
)
# Skip header
next(fp)
reader = csv.reader(fp)
for row in reader:
raw_code, gc, raw_lower, raw_upper, raw_title, raw_islower, raw_isupper = row
char = parse_codepoint(raw_code)
codepoint = f"U+{raw_code.upper():0>4}"
check(gc == general_category(char), (char, gc, general_category(char)))
check_case_mapping("lower", char, raw_lower, verbosity)
check_case_mapping("upper", char, raw_upper, verbosity)
check_case_mapping("title", char, raw_title, verbosity)
check_case_predicate("islower", char, raw_islower, verbosity)
check_case_predicate("isupper", char, raw_isupper, verbosity)
|
