1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
# Copyright (C) 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import errno
import glob
import json
import logging
import os
import re
import bottle
import cliapp
import lorrycontroller
class ReadConfiguration(lorrycontroller.LorryControllerRoute):
http_method = 'POST'
path = '/1.0/read-configuration'
def run(self, **kwargs):
logging.info('%s %s called', self.http_method, self.path)
self.get_confgit()
conf_obj = self.read_config_file()
self.fix_up_parsed_fields(conf_obj)
statedb = self.open_statedb()
with statedb:
existing_lorries = set(statedb.get_lorries_paths())
existing_troves = set(statedb.get_troves())
for section in conf_obj:
if section['type'] == 'lorries':
added = self.add_matching_lorries_to_statedb(
statedb, section)
existing_lorries = existing_lorries.difference(added)
elif section['type'] in ('trove', 'troves'):
self.add_trove(statedb, section)
if section['trovehost'] in existing_troves:
existing_troves.remove(section['trovehost'])
existing_lorries = self.without_lorries_for_trovehost(
statedb, existing_lorries, section['trovehost'])
else:
logging.warning(
'Unknown section in configuration: %r', section)
for path in existing_lorries:
statedb.remove_lorry(path)
for trovehost in existing_troves:
statedb.remove_trove(trovehost)
statedb.remove_lorries_for_trovehost(trovehost)
if 'redirect' in bottle.request.forms:
bottle.redirect(bottle.request.forms.redirect)
return 'Configuration has been updated.'
def without_lorries_for_trovehost(self, statedb, lorries, trovehost):
for_trovehost = statedb.get_lorries_for_trove(trovehost)
return set(x for x in lorries if x not in for_trovehost)
def get_confgit(self):
if self.app_settings['debug-real-confgit']:
confdir = self.app_settings['configuration-directory']
if not os.path.exists(confdir):
self.git_clone_confgit(confdir)
else:
self.git_pull_confgit(confdir)
def git_clone_confgit(self, confdir):
url = self.app_settings['confgit-url']
branch = self.app_settings['confgit-branch']
cliapp.runcmd(['git', 'clone', '-b', branch, url, confdir])
def git_pull_confgit(self, confdir):
cliapp.runcmd(['git', 'pull'], cwd=confdir)
@property
def config_file_name(self):
return os.path.join(
self.app_settings['configuration-directory'],
'lorry-controller.conf')
def read_config_file(self):
'''Read the configuration file, return as Python object.'''
filename = self.config_file_name
try:
with open(filename) as f:
return json.load(f)
except IOError as e:
if e.errno == errno.ENOENT:
# File doesn't exist. Return an empty configuration.
return []
bottle.abort(500, 'Error reading %s: %s' % (filename, e))
def fix_up_parsed_fields(self, obj):
for item in obj:
item['interval'] = self.fix_up_interval(item.get('interval'))
item['ls-interval'] = self.fix_up_interval(item.get('ls-interval'))
def fix_up_interval(self, value):
default_interval = 86400 # 1 day
if not value:
return default_interval
m = re.match('(\d+)\s*(s|m|h|d)?', value, re.I)
if not m:
return default_value
number, factor = m.groups()
factors = {
's': 1,
'm': 60,
'h': 60*60,
'd': 60*60*24,
}
if factor is None:
factor = 's'
factor = factors.get(factor.lower(), 1)
return int(number) * factor
def add_matching_lorries_to_statedb(self, statedb, section):
added_paths = set()
filenames = self.find_lorry_files_for_section(section)
lorry_specs = []
for filename in sorted(filenames):
for lorry_spec in self.get_lorry_specs(filename):
self.add_refspecs_if_missing(lorry_spec)
lorry_specs.append(lorry_spec)
for lorry_spec in sorted(lorry_specs):
path = self.deduce_repo_path(section, lorry_spec)
text = self.serialise_lorry_spec(lorry_spec)
interval = section['interval']
try:
old_lorry_info = statedb.get_lorry_info(path)
except lorrycontroller.LorryNotFoundError:
old_lorry_info = None
statedb.add_to_lorries(
path=path, text=text, from_trovehost='', from_path='',
interval=interval)
added_paths.add(path)
return added_paths
def find_lorry_files_for_section(self, section):
result = []
dirname = os.path.dirname(self.config_file_name)
for base_pattern in section['globs']:
pattern = os.path.join(dirname, base_pattern)
result.extend(glob.glob(pattern))
return result
def get_lorry_specs(self, filename):
with open(filename) as f:
obj = json.load(f)
return obj.items()
def add_refspecs_if_missing(self, lorry_spec):
base_path, details = lorry_spec
if 'refspecs' not in details:
details['refspecs'] = [
'+refs/heads/*',
'+refs/tags/*',
]
def deduce_repo_path(self, section, lorry_spec):
base_path, details = lorry_spec
return '%s/%s' % (section['prefix'], base_path)
def serialise_lorry_spec(self, lorry_spec):
key, details = lorry_spec
obj = { key: details }
return json.dumps(obj, indent=4)
def add_trove(self, statedb, section):
statedb.add_trove(
trovehost=section['trovehost'],
lorry_interval=section['interval'],
ls_interval=section['ls-interval'],
prefixmap=json.dumps(section['prefixmap']),
ignore=json.dumps(section['ignore']))
|
