Add support for multiple raw files per lorry

- This allows consumer to group raw files so that
  they end up in the same mirrored lfs repository
- Useful for raw file mirroring scenarios where a
  common shared prefix is desired (e.g. for a
  BuildStream project)
- Use "urls" key for consistency with other lorry
  types (all use "url")

1 files changed, 35 insertions, 29 deletions

diff --git a/lorry b/lorry
index 4a15e83..1f4c026 100755
--- a/lorry
+++ b/lorry
@@ -796,33 +796,39 @@ class Lorry(cliapp.Application):
     def gitify_raw_file(self, project_name, dirname, gitdir, spec):
         raw_file_branch = 'master'
         raw_file_refspecs = 'refs/heads/{branch}:refs/heads/{branch}'.format(branch=raw_file_branch)
-        # Fetch the file
-        url = spec['url']
-        url_path = urllib.parse.urlparse(url)[2]
-        basename = os.path.basename(url_path)
-        file_dest = os.path.join(dirname, basename)
-        self.progress('.. checking if we need to fetch %s' % basename)
-        if file_missing_or_empty(file_dest):
-            self.progress('.. attempting to fetch.')
-            try:
-                with open(file_dest, 'wb') as raw_file:
-                    urlfile = urllib.request.urlopen(spec['url'])
-                    raw_file.write(urlfile.read())
-                    try:
-                        # HTTP dates use (one of) the email date formats
-                        url_date = email.utils.mktime_tz(
-                            email.utils.parsedate_tz(
-                                urlfile.info()['Last-Modified']))
-                    except (KeyError, ValueError, TypeError):
-                        url_date = None
-                    urlfile.close()
-                if url_date:
-                    os.utime(file_dest, (url_date, url_date))
-            except Exception:
-                if os.path.exists(file_dest):
-                    os.unlink(file_dest)
-                raise
-        else:
+        # Fetch the files
+        new_files = {}
+        for src in spec['urls']:
+            url = spec['urls'][src]
+            url_path = urllib.parse.urlparse(url)[2]
+            basename = os.path.basename(url_path)
+            file_dest = os.path.join(dirname, basename)
+            self.progress('.. checking if we need to fetch %s' % basename)
+            if file_missing_or_empty(file_dest):
+                new_files[src] = file_dest
+                self.progress('.. attempting to fetch %s' % basename)
+                try:
+                    with open(file_dest, 'wb') as raw_file:
+                        urlfile = urllib.request.urlopen(url)
+                        raw_file.write(urlfile.read())
+                        try:
+                            # HTTP dates use (one of) the email date formats
+                            url_date = email.utils.mktime_tz(
+                                email.utils.parsedate_tz(
+                                    urlfile.info()['Last-Modified']))
+                        except (KeyError, ValueError, TypeError):
+                            url_date = None
+                        urlfile.close()
+                    if url_date:
+                        os.utime(file_dest, (url_date, url_date))
+                except Exception:
+                    if os.path.exists(file_dest):
+                        os.unlink(file_dest)
+                    raise
+            else:
+                self.progress('nothing to do for %s' % basename)
+
+        if not len(new_files):
             self.progress('.. no need to run, nothing to do')
             return
 
@@ -838,7 +844,8 @@ class Lorry(cliapp.Application):
         # Ensure the repo supports git LFS
         self.run_program(['git', 'lfs', 'install'], cwd=gitdir)
 
-        self.run_program(["%s.raw-file-importer" % lorry_path, file_dest, project_name], cwd=gitdir)
+        for src in new_files:
+            self.run_program(["%s.raw-file-importer" % lorry_path, new_files[src], src], cwd=gitdir)
 
     def gitify_archive(self, archive_type, project_name, dirname, gitdir, spec):
         assert archive_type in ['zip', 'gzip', 'tar']
@@ -981,4 +988,3 @@ class Lorry(cliapp.Application):
 
 if __name__ == '__main__':
     Lorry(version=__version__).run()
-