1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
#
# Copyright (C) 2016 Codethink Limited
# Copyright (C) 2019 Bloomberg L.P.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
# Daniel Silverstone <daniel.silverstone@codethink.co.uk>
import re
import sys
from ._exceptions import LoadError, LoadErrorReason
from . import _yaml
# Variables are allowed to have dashes here
#
PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
# The Variables helper object will resolve the variable references in
# the given dictionary, expecting that any dictionary values which contain
# variable references can be resolved from the same dictionary.
#
# Each Element creates its own Variables instance to track the configured
# variable settings for the element.
#
# Args:
# node (dict): A node loaded and composited with yaml tools
#
# Raises:
# LoadError, if unresolved variables, or cycles in resolution, occur.
#
class Variables():
def __init__(self, node):
self.original = node
self.newexp = self._resolve(node)
self.flat = self._flatten()
# subst():
#
# Substitutes any variables in 'string' and returns the result.
#
# Args:
# (string): The string to substitute
#
# Returns:
# (string): The new string with any substitutions made
#
# Raises:
# LoadError, if the string contains unresolved variable references.
#
def subst(self, string):
exp = _parse_expstr(string)
try:
return _expand_expstr(self.newexp, exp)
except KeyError:
unmatched = []
for v in exp[1][1::2]:
if v not in self.newexp:
unmatched.append(v)
if unmatched:
if len(unmatched) == 1:
message = "Unresolved variable '{var}'".format(var=unmatched[0])
else:
message = "Unresolved variables: "
for unmatch in unmatched:
if unmatched.index(unmatch) > 0:
message += ', '
message += unmatch
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
raise
# Variable resolving code
#
# Here we resolve all of our inputs into a dictionary, ready for use
# in subst()
def _resolve(self, node):
# Special case, if notparallel is specified in the variables for this
# element, then override max-jobs to be 1.
# Initialize it as a string as all variables are processed as strings.
#
if _yaml.node_get(node, bool, 'notparallel', default_value=False):
node['max-jobs'] = str(1)
ret = {}
for key, value in _yaml.node_items(node):
value = _yaml.node_get(node, str, key)
ret[sys.intern(key)] = _parse_expstr(value)
return ret
def _check_for_missing(self):
# First the check for anything unresolvable
summary = []
for key, expstr in self.newexp.items():
for var in expstr[1][1::2]:
if var not in self.newexp:
line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}"
provenance = _yaml.node_get_provenance(self.original, key)
summary.append(line.format(unmatched=var, variable=key, provenance=provenance))
if summary:
raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
"Failed to resolve one or more variable:\n{}\n".format("\n".join(summary)))
def _check_for_cycles(self):
# And now the cycle checks
def cycle_check(exp, visited, cleared):
for var in exp[1][1::2]:
if var in cleared:
continue
if var in visited:
raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
"{}: ".format(_yaml.node_get_provenance(self.original, var)) +
("Variable '{}' expands to contain a reference to itself. " +
"Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var))
visited.append(var)
cycle_check(self.newexp[var], visited, cleared)
visited.pop()
cleared.add(var)
cleared = set()
for key, expstr in self.newexp.items():
if key not in cleared:
cycle_check(expstr, [key], cleared)
# _flatten():
#
# Turn our dictionary of expansion strings into a flattened set
# so that we can run expansions faster in the future
#
# Raises:
# LoadError, if the string contains unresolved variable references or
# if cycles are detected in the variable references
#
def _flatten(self):
flat = {}
try:
for key, expstr in self.newexp.items():
if expstr[0] > 1:
expstr = (1, [sys.intern(_expand_expstr(self.newexp, expstr))])
self.newexp[key] = expstr
flat[key] = expstr[1][0]
except KeyError:
self._check_for_missing()
raise
except RecursionError:
self._check_for_cycles()
raise
return flat
# Cache for the parsed expansion strings. While this is nominally
# something which might "waste" memory, in reality each of these
# will live as long as the element which uses it, which is the
# vast majority of the memory usage across the execution of BuildStream.
PARSE_CACHE = {
# Prime the cache with the empty string since otherwise that can
# cause issues with the parser, complications to which cause slowdown
"": (1, [""]),
}
# Helper to parse a string into an expansion string tuple, caching
# the results so that future parse requests don't need to think about
# the string
def _parse_expstr(instr):
try:
return PARSE_CACHE[instr]
except KeyError:
# This use of the regex turns a string like "foo %{bar} baz" into
# a list ["foo ", "bar", " baz"]
splits = PARSE_EXPANSION.split(instr)
# If an expansion ends the string, we get an empty string on the end
# which we can optimise away, making the expansion routines not need
# a test for this.
if splits[-1] == '':
splits = splits[:-1]
# Cache an interned copy of this. We intern it to try and reduce the
# memory impact of the cache. It seems odd to cache the list length
# but this is measurably cheaper than calculating it each time during
# string expansion.
PARSE_CACHE[instr] = (len(splits), [sys.intern(s) for s in splits])
return PARSE_CACHE[instr]
# Helper to expand a given top level expansion string tuple in the context
# of the given dictionary of expansion strings.
#
# Note: Will raise KeyError if any expansion is missing
def _expand_expstr(content, topvalue):
if topvalue[0] == 1:
return topvalue[1][0]
def __expand(value):
(expansion_len, expansion_bits) = value
idx = 0
while idx < expansion_len:
# First yield any constant string content
yield expansion_bits[idx]
idx += 1
# Now, if there is an expansion variable left to expand, yield
# the expansion of that variable too
if idx < expansion_len:
yield from __expand(content[expansion_bits[idx]])
idx += 1
return "".join(__expand(topvalue))
|
