1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
# Copyright (C) 2012-2015 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import cliapp
import logging
import urllib
import urlparse
import requests
class GetError(cliapp.AppException):
def __init__(self, cache, filename, http_error):
self.http_error = http_error
cliapp.AppException.__init__(
self, 'Failed to get the file %s from the artifact cache %s: %s' %
(filename, cache, http_error))
class RemoteArtifactCache(object):
def __init__(self, server_url):
self.server_url = server_url
def has(self, artifact):
return self._has_file(artifact.basename())
def _has_file(self, filename):
url = self._request_url(filename)
logging.debug('RemoteArtifactCache._has_file: url=%s' % url)
response = requests.head(url)
if response.status_code == 404:
return False
elif response.ok:
return True
else:
response.raise_for_status()
def _request_url(self, filename):
server_url = self.server_url
if not server_url.endswith('/'):
server_url += '/'
return urlparse.urljoin(
server_url, '/1.0/artifacts?filename=%s' %
urllib.quote(filename))
def __str__(self): # pragma: no cover
return self.server_url
def _fetch_file(self, remote_filename, local_file, status_cb=None,
error_if_missing=True):
chunk_size = 10 * 1024 * 1024
def show_status(downloaded, total):
if not status_cb:
return
downloaded = min(downloaded, total)
if total == 0:
status_cb(msg='%(file)s: Fetched %(d).02fMB',
file=remote_filename,
d=max(downloaded / (1024 * 1024), 0.01),
chatty=True)
else:
status_cb(msg='%(file)s: Fetched %(d).02fMB of %(t).02fMB',
file=remote_filename,
d=max(downloaded / (1024 * 1024), 0.01),
t=max(total / (1024 * 1024), 0.01),
chatty=True)
remote_url = self._request_url(remote_filename)
logging.debug('RemoteArtifactCache._fetch_file: url=%s' % remote_url)
try:
response = requests.get(remote_url, stream=True)
response.raise_for_status()
content_length = int(response.headers.get('content-length', 0))
for i, chunk in enumerate(response.iter_content(chunk_size)):
local_file.write(chunk)
show_status((i+1) * chunk_size, content_length)
except requests.exceptions.HTTPError as e:
logging.debug(str(e))
if e.response.status_code != 404 or error_if_missing:
raise GetError(self, remote_filename, e)
def _fetch_files(self, to_fetch, status_cb):
'''Fetch a set of files atomically.
If an error occurs during the transfer of any files, all downloaded
data is deleted, to reduce the chances of having artifacts in the local
cache that are missing their metadata, and so on.
This assumes that the morphlib.savefile module is used so the file
handles passed in to_fetch have a .abort() method.
'''
def is_required(basename):
# This is a workaround for a historical bugs. Distbuild used to
# fail to transfer the source .meta and the .build-log file, so
# we need to cope if they are missing.
if basename.endswith('.build-log') or basename.endswith('.meta'):
return False
return True
try:
for remote_filename, local_file in to_fetch:
self._fetch_file(
remote_filename, local_file, status_cb=status_cb,
error_if_missing=is_required(remote_filename))
except BaseException:
for _, local_file in to_fetch:
local_file.abort()
raise
else:
for _, local_file in to_fetch:
local_file.close()
def get_artifacts_for_source(self, source, lac, status_cb=None):
'''Ensure all built artifacts for 'source' are in the local cache.
This includes all build metadata such as the .build-log file.
'''
to_fetch = []
for basename in source.files():
if not lac.has_file(basename):
to_fetch.append((basename, lac.put_file(basename)))
if len(to_fetch) > 0:
if status_cb:
status_cb(
msg='Fetching built artifacts of %(name)s',
name=source.name)
self._fetch_files(to_fetch, status_cb=status_cb)
def get_artifacts_for_sources(self, sources, lac, status_cb=None):
'''Ensure artifacts for multiple sources are available locally.'''
# Running the downloads in parallel might give a speed boost, as many
# of these are small files.
for source in sources:
self.get_artifacts_for_source(source, lac, status_cb=status_cb)
|
