diff options
Diffstat (limited to 'lorry')
| -rwxr-xr-x | lorry | 57 |
1 files changed, 55 insertions, 2 deletions
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (C) 2011-2020 Codethink Limited +# Copyright (C) 2011-2021 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -343,6 +343,7 @@ class Lorry(cliapp.Application): 'git': self.mirror_git, 'hg': self.gitify_hg, 'svn': self.gitify_svn, + 'raw-file': self.gitify_raw_file, 'tarball': functools.partial(self.gitify_archive, 'tar'), 'zip': functools.partial(self.gitify_archive, 'zip'), 'gzip': functools.partial(self.gitify_archive, 'gzip') @@ -792,6 +793,59 @@ class Lorry(cliapp.Application): *plugin_options], cwd=gitdir) + def gitify_raw_file(self, project_name, dirname, gitdir, spec): + raw_file_branch = 'master' + raw_file_refspecs = 'refs/heads/{branch}:refs/heads/{branch}'.format(branch=raw_file_branch) + # Fetch the files + new_files = {} + for src in spec['urls']: + url = spec['urls'][src] + url_path = urllib.parse.urlparse(url)[2] + basename = os.path.basename(url_path) + file_dest = os.path.join(dirname, basename) + self.progress('.. checking if we need to fetch %s' % basename) + if file_missing_or_empty(file_dest): + new_files[src] = file_dest + self.progress('.. attempting to fetch %s' % basename) + try: + with open(file_dest, 'wb') as raw_file, \ + urllib.request.urlopen(url) as urlfile: + raw_file.write(urlfile.read()) + try: + # HTTP dates use (one of) the email date formats + url_date = email.utils.mktime_tz( + email.utils.parsedate_tz( + urlfile.info()['Last-Modified'])) + except (KeyError, ValueError, TypeError): + url_date = None + if url_date: + os.utime(file_dest, (url_date, url_date)) + except Exception: + if os.path.exists(file_dest): + os.unlink(file_dest) + raise + else: + self.progress('nothing to do for %s' % basename) + + if not len(new_files): + self.progress('.. no need to run, nothing to do') + return + + self.ensure_gitdir(gitdir) + # Ensure the repo is up-to-date + pullurl = "%s/%s.git" % (self.settings['mirror-base-url-push'], project_name) + try: + self.run_program(['git', 'fetch', pullurl, raw_file_refspecs], cwd=gitdir) + except: + # TODO: Be more specific about which exceptions are fine + pass + + # Ensure the repo supports git LFS + self.run_program(['git', 'lfs', 'install', '--local'], cwd=gitdir) + + for src in new_files: + self.run_program(["%s.raw-file-importer" % lorry_path, new_files[src], src], cwd=gitdir) + def gitify_archive(self, archive_type, project_name, dirname, gitdir, spec): assert archive_type in ['zip', 'gzip', 'tar'] @@ -933,4 +987,3 @@ class Lorry(cliapp.Application): if __name__ == '__main__': Lorry(version=__version__).run() - |