1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
|
# Copyright (C) 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import cliapp
import morphlib
import six
import json
import logging
import os
class LorrySetError(cliapp.AppException):
pass
class DuplicateLorryError(LorrySetError):
def __init__(self, filename, existing_lorry_name, existing_lorry_file):
message = "%s: duplicates existing lorry '%s' from file %s" % (
os.path.relpath(filename), existing_lorry_name,
os.path.relpath(existing_lorry_file))
super(DuplicateLorryError, self).__init__(message)
class LorrySet(object):
'''Manages a set of .lorry files.
A LorrySet instance operates on all the .lorry files inside the path given
at construction time. This includes .lorry files in subdirectories of the
path.
A lorry *entry* describes the upstream repository for a given project,
which is associated to a name such as 'ruby-gems/chef' or 'gcc-tarball'. A
lorry *file* contains one or more *entries*. The filename of a lorry file
is not necessarily linked to the name of any lorry entry that it contains.
'''
def __init__(self, lorries_path):
'''Initialise a LorrySet instance for the given directory.
This will load and parse all of the .lorry files inside 'lorries_path'
into memory.
'''
self.path = lorries_path
if os.path.exists(lorries_path):
self.data = self._parse_all_lorries()
else:
os.makedirs(lorries_path)
self.data = {}
def all_lorry_files(self):
'''Return the path of each lorry file in this set.'''
for dirpath, dirnames, filenames in os.walk(self.path):
for filename in filenames:
if filename.endswith('.lorry'):
yield os.path.join(dirpath, filename)
def _parse_all_lorries(self):
lorry_set = {}
# We keep track of which entry came from which file, but only while
# loading everything into memory (to allow us to give more helpful
# errors).
filenames = {}
for lorry_file in self.all_lorry_files():
lorry = self._parse_lorry(lorry_file)
lorry_items = lorry.items()
for key, value in lorry_items:
if key in lorry_set:
raise DuplicateLorryError(lorry_file, key, filenames[key])
filenames[key] = lorry_file
lorry_set.update(lorry_items)
return lorry_set
def _parse_lorry(self, lorry_file):
try:
with open(lorry_file, 'r') as f:
lorry = json.load(f)
return lorry
except ValueError as e:
raise LorrySetError(
"Error parsing %s: %s" % (lorry_file, e))
def get_lorry(self, name):
'''Return the lorry entry for the named project.'''
return {name: self.data[name]}
def find_lorry_for_package(self, kind, package_name):
'''Find the lorry entry for a given foreign package, or return None.
This makes use of an extension to the .lorry format made by the
Baserock Import tool. Fields follow the form 'x-products-$KIND'
and specify the name of a package in the foreign packaging universe
named $KIND.
'''
key = 'x-products-%s' % kind
for name, lorry in self.data.iteritems():
products = lorry.get(key, [])
for entry in products:
if entry == package_name:
return {name: lorry}
return None
def _check_for_conflicts_in_standard_fields(self, existing, new):
'''Ensure that two lorries for the same project do actually match.'''
for field, value in existing.iteritems():
if field.startswith('x-'):
continue
if field == 'url':
# FIXME: need a much better way of detecting whether the URLs
# are equivalent ... right now HTTP vs. HTTPS will cause an
# error, for example!
matches = (value.rstrip('/') == new[field].rstrip('/'))
else:
matches = (value == new[field])
if not matches:
raise LorrySetError(
'Lorry %s conflicts with existing entry %s at field %s' %
(new, existing, field))
def _merge_products_fields(self, existing, new):
'''Merge the x-products- fields from new lorry into an existing one.'''
is_product_field = lambda x: x.startswith('x-products-')
existing_fields = [f for f in existing.iterkeys() if
is_product_field(f)]
new_fields = [f for f in new.iterkeys() if f not in existing_fields and
is_product_field(f)]
for field in existing_fields:
existing[field].extend(new[field])
existing[field] = list(set(existing[field]))
for field in new_fields:
existing[field] = new[field]
def _add_lorry_entry_to_lorry_file(self, filename, entry):
if os.path.exists(filename):
with open(filename) as f:
contents = json.load(f)
else:
contents = {}
contents.update(entry)
with morphlib.savefile.SaveFile(filename, 'w') as f:
json.dump(contents, f, indent=4, separators=(',', ': '),
sort_keys=True)
def add(self, filename, lorry_entry):
'''Add a lorry entry to the named .lorry file.
The lorry_entry should follow the on-disk format for a lorry stanza,
which is a dict of one entry mapping the name of the entry to its
contents.
The .lorry file will be created if it doesn't exist.
'''
logging.debug('Adding %s to lorryset', filename)
filename = os.path.join(self.path, '%s.lorry' % filename)
assert len(lorry_entry) == 1
project_name = lorry_entry.keys()[0]
info = lorry_entry.values()[0]
if len(project_name) == 0:
raise LorrySetError(
'Invalid lorry %s: %s' % (filename, lorry_entry))
if not isinstance(info.get('url'), six.string_types):
raise LorrySetError(
'Invalid URL in lorry %s: %s' % (filename, info.get('url')))
if project_name in self.data:
stored_lorry = self.get_lorry(project_name)
self._check_for_conflicts_in_standard_fields(
stored_lorry[project_name], lorry_entry[project_name])
self._merge_products_fields(
stored_lorry[project_name], lorry_entry[project_name])
lorry_entry = stored_lorry
else:
self.data[project_name] = info
self._add_lorry_entry_to_lorry_file(filename, lorry_entry)
|
