diff options
| -rwxr-xr-x | lorry | 64 |
1 files changed, 35 insertions, 29 deletions
@@ -796,33 +796,39 @@ class Lorry(cliapp.Application): def gitify_raw_file(self, project_name, dirname, gitdir, spec): raw_file_branch = 'master' raw_file_refspecs = 'refs/heads/{branch}:refs/heads/{branch}'.format(branch=raw_file_branch) - # Fetch the file - url = spec['url'] - url_path = urllib.parse.urlparse(url)[2] - basename = os.path.basename(url_path) - file_dest = os.path.join(dirname, basename) - self.progress('.. checking if we need to fetch %s' % basename) - if file_missing_or_empty(file_dest): - self.progress('.. attempting to fetch.') - try: - with open(file_dest, 'wb') as raw_file: - urlfile = urllib.request.urlopen(spec['url']) - raw_file.write(urlfile.read()) - try: - # HTTP dates use (one of) the email date formats - url_date = email.utils.mktime_tz( - email.utils.parsedate_tz( - urlfile.info()['Last-Modified'])) - except (KeyError, ValueError, TypeError): - url_date = None - urlfile.close() - if url_date: - os.utime(file_dest, (url_date, url_date)) - except Exception: - if os.path.exists(file_dest): - os.unlink(file_dest) - raise - else: + # Fetch the files + new_files = {} + for src in spec['urls']: + url = spec['urls'][src] + url_path = urllib.parse.urlparse(url)[2] + basename = os.path.basename(url_path) + file_dest = os.path.join(dirname, basename) + self.progress('.. checking if we need to fetch %s' % basename) + if file_missing_or_empty(file_dest): + new_files[src] = file_dest + self.progress('.. attempting to fetch %s' % basename) + try: + with open(file_dest, 'wb') as raw_file: + urlfile = urllib.request.urlopen(url) + raw_file.write(urlfile.read()) + try: + # HTTP dates use (one of) the email date formats + url_date = email.utils.mktime_tz( + email.utils.parsedate_tz( + urlfile.info()['Last-Modified'])) + except (KeyError, ValueError, TypeError): + url_date = None + urlfile.close() + if url_date: + os.utime(file_dest, (url_date, url_date)) + except Exception: + if os.path.exists(file_dest): + os.unlink(file_dest) + raise + else: + self.progress('nothing to do for %s' % basename) + + if not len(new_files): self.progress('.. no need to run, nothing to do') return @@ -838,7 +844,8 @@ class Lorry(cliapp.Application): # Ensure the repo supports git LFS self.run_program(['git', 'lfs', 'install'], cwd=gitdir) - self.run_program(["%s.raw-file-importer" % lorry_path, file_dest, project_name], cwd=gitdir) + for src in new_files: + self.run_program(["%s.raw-file-importer" % lorry_path, new_files[src], src], cwd=gitdir) def gitify_archive(self, archive_type, project_name, dirname, gitdir, spec): assert archive_type in ['zip', 'gzip', 'tar'] @@ -981,4 +988,3 @@ class Lorry(cliapp.Application): if __name__ == '__main__': Lorry(version=__version__).run() - |