1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Helpers for dealing with translation files."""
import ast
import os
import re
import xml.etree.cElementTree as ElementTree
class GRDFile(object):
"""Class representing a grd xml file.
Attributes:
path: the path to the grd file.
dir: the path to the the grd's parent directery.
name: the base name of the grd file.
grdp_paths: the list of grdp files included in the grd via <part>.
structure_paths: the paths of any <structure> elements in the grd file.
xtb_paths: the xtb paths where the grd's translations live.
lang_to_xtb_path: maps each language to the xtb path for that language.
appears_translatable: whether the contents of the grd indicate that it's
supposed to be translated.
expected_languages: the languages that this grd is expected to have
translations for, based on the translation expectations file.
"""
def __init__(self, path):
self.path = path
self.dir, self.name = os.path.split(path)
dom, self.grdp_paths = _parse_grd_file(path)
self.structure_paths = [os.path.join(self.dir, s.get('file'))
for s in dom.findall('.//structure')]
self.xtb_paths = [os.path.join(self.dir, f.get('path'))
for f in dom.findall('.//file')]
self.lang_to_xtb_path = {}
self.appears_translatable = (len(self.xtb_paths) != 0 or
dom.find('.//message') is not None)
self.expected_languages = None
def _populate_lang_to_xtb_path(self, errors):
"""Populates the lang_to_xtb_path attribute."""
grd_root = os.path.splitext(self.name)[0]
lang_pattern = re.compile(r'%s_([^_]+)\.xtb$' % re.escape(grd_root))
for xtb_path in self.xtb_paths:
xtb_basename = os.path.basename(xtb_path)
xtb_lang_match = re.match(lang_pattern, xtb_basename)
if not xtb_lang_match:
errors.append('%s: invalid xtb name: %s. xtb name must be %s_<lang>'
'.xtb where <lang> is the language code.' %
(self.name, xtb_basename, grd_root))
continue
xtb_lang = xtb_lang_match.group(1)
if xtb_lang in self.lang_to_xtb_path:
errors.append('%s: %s is listed twice' % (self.name, xtb_basename))
continue
self.lang_to_xtb_path[xtb_lang] = xtb_path
return errors
def get_translatable_grds(repo_root, all_grd_paths,
translation_expectations_path):
"""Returns all the grds that should be translated as a list of GRDFiles.
This verifies that every grd file that appears translatable is listed in
the translation expectations, and that every grd in the translation
expectations actually exists.
Args:
repo_root: The path to the root of the repository.
all_grd_paths: All grd paths in the repository relative to repo_root.
translation_expectations_path: The path to the translation expectations
file, which specifies which grds to translate and into which languages.
"""
grd_to_langs, untranslated_grds = _parse_translation_expectations(
translation_expectations_path)
# Check that every grd that appears translatable is listed in
# the translation expectations.
grds_with_expectations = set(grd_to_langs.keys()).union(untranslated_grds)
all_grds = {p: GRDFile(os.path.join(repo_root, p)) for p in all_grd_paths}
errors = []
for path, grd in all_grds.iteritems():
if grd.appears_translatable:
if path not in grds_with_expectations:
errors.append('%s appears to be translatable (because it contains '
'<file> or <message> elements), but is not listed in the '
'translation expectations.' % path)
# Check that every file in translation_expectations exists.
for path in grds_with_expectations:
if path not in all_grd_paths:
errors.append('%s is listed in the translation expectations, but this '
'grd file does not exist.' % path)
if errors:
raise Exception('%s needs to be updated. Please fix these issues:\n - %s' %
(translation_expectations_path, '\n - '.join(errors)))
translatable_grds = []
for path, expected_languages_list in grd_to_langs.iteritems():
grd = all_grds[path]
grd.expected_languages = expected_languages_list
grd._populate_lang_to_xtb_path(errors)
translatable_grds.append(grd)
# Ensure each grd lists the expected languages.
expected_languages = set(expected_languages_list)
actual_languages = set(grd.lang_to_xtb_path.keys())
if expected_languages.difference(actual_languages):
errors.append('%s: missing translations for these languages: %s. Add '
'<file> and <output> elements to the grd file, or update '
'the translation expectations.' % (grd.name,
sorted(expected_languages.difference(actual_languages))))
if actual_languages.difference(expected_languages):
errors.append('%s: references translations for unexpected languages: %s. '
'Remove the offending <file> and <output> elements from the'
' grd file, or update the translation expectations.'
% (grd.name,
sorted(actual_languages.difference(expected_languages))))
if errors:
raise Exception('Please fix these issues:\n - %s' %
('\n - '.join(errors)))
return translatable_grds
def _parse_grd_file(grd_path):
"""Reads a grd(p) file and any subfiles included via <part file="..." />.
Args:
grd_path: The path of the .grd or .grdp file.
Returns:
A tuple (grd_dom, grdp_paths). dom is an ElementTree DOM for the grd file,
with the <part> elements inlined. grdp_paths is the list of grdp files that
were included via <part> elements.
"""
grdp_paths = []
grd_dom = ElementTree.parse(grd_path)
# We modify grd in the loop, so listify this iterable to be safe.
part_nodes = list(grd_dom.findall('.//part'))
for part_node in part_nodes:
grdp_rel_path = part_node.get('file')
grdp_path = os.path.join(os.path.dirname(grd_path), grdp_rel_path)
grdp_paths.append(grdp_path)
grdp_dom, grdp_grpd_paths = _parse_grd_file(grdp_path)
grdp_paths.extend(grdp_grpd_paths)
part_node.append(grdp_dom.getroot())
return grd_dom, grdp_paths
def _parse_translation_expectations(path):
"""Parses a translations expectations file.
Example translations expectations file:
{
"desktop_grds": {
"languages": ["es", "fr"],
"files": [
"ash/ash_strings.grd",
"ui/strings/ui_strings.grd",
],
},
"android_grds": {
"languages": ["de", "pt-BR"],
"files": [
"chrome/android/android_chrome_strings.grd",
},
"untranslated_grds": {
"chrome/locale_settings.grd": "Not UI strings; localized separately",
"chrome/locale_settings_mac.grd": "Not UI strings; localized separately",
},
}
Returns:
A tuple (grd_to_langs, untranslated_grds). grd_to_langs maps each grd path
to the list of languages into which that grd should be translated.
untranslated_grds is a list of grds that "appear translatable" but should
not be translated.
"""
with open(path) as f:
file_contents = f.read()
def assert_list_of_strings(l, name):
assert isinstance(l, list) and all(isinstance(s, basestring) for s in l), (
'%s must be a list of strings' % name)
try:
translations_expectations = ast.literal_eval(file_contents)
assert isinstance(translations_expectations, dict), (
'%s must be a python dict' % path)
grd_to_langs = {}
untranslated_grds = []
for group_name, settings in translations_expectations.iteritems():
if group_name == 'untranslated_grds':
untranslated_grds = list(settings.keys())
assert_list_of_strings(untranslated_grds, 'untranslated_grds')
continue
languages = settings['languages']
files = settings['files']
assert_list_of_strings(languages, group_name + '.languages')
assert_list_of_strings(files, group_name + '.files')
for grd in files:
grd_to_langs[grd] = languages
return grd_to_langs, untranslated_grds
except Exception:
print 'Error: failed to parse', path
raise
|
