Merge branch 'shprdchris/raw-file-new-format' into 'master'

(v2) Make raw importer honour multiple files

Closes #29

See merge request CodethinkLabs/lorry/lorry!33

2 files changed, 15 insertions, 9 deletions

diff --git a/README.md b/README.md
index 199e8da..302b443 100644
--- a/README.md
+++ b/README.md
@@ -297,10 +297,16 @@ repository.
     {
         "raw-file-repo": {
             "type": "raw-file",
-            "urls": {
-                "authorities/england-and-wales": "http://geoportal1-ons.opendata.arcgis.com/datasets/0b09996863af4b5db78058225bac5d1b_0.kml",
-                "radiological-monitoring-data": "https://fsadata.github.io/radiological-monitoring-data/data/provisional-2016-bi-annual-aquatic-and-terrestrial-monitoring-results-26062017-raw-results.csv"
-            }
+            "urls": [
+                {
+                    "url": "http://geoportal1-ons.opendata.arcgis.com/datasets/0b09996863af4b5db78058225bac5d1b_0.kml",
+                    "destination": "authorities/england-and-wales"
+                },
+                {
+                    "url": "https://fsadata.github.io/radiological-monitoring-data/data/provisional-2016-bi-annual-aquatic-and-terrestrial-monitoring-results-26062017-raw-results.csv",
+                    "destination": "radiological-monitoring-data"
+                }
+            ]
         }
     }
 
diff --git a/lorry b/lorry
index bc97350..b52b563 100755
--- a/lorry
+++ b/lorry
@@ -805,15 +805,15 @@ class Lorry(cliapp.Application):
         raw_file_refspecs = 'refs/heads/{branch}:refs/heads/{branch}'.format(branch=raw_file_branch)
         self.ensure_gitdir(gitdir)
         # Fetch the files
-        new_files = {}
+        new_files = []
         for src in spec['urls']:
-            url = spec['urls'][src]
+            url = src['url']
             url_path = urllib.parse.urlparse(url)[2]
             basename = os.path.basename(url_path)
             file_dest = os.path.join(dirname, basename)
             self.progress('.. checking if we need to fetch %s' % basename)
             if file_missing_or_empty(file_dest):
-                new_files[src] = file_dest
+                new_files.append((src['destination'], file_dest))
                 self.progress('.. attempting to fetch %s' % basename)
                 try:
                     with open(file_dest, 'wb') as raw_file, \
@@ -850,8 +850,8 @@ class Lorry(cliapp.Application):
         # Ensure the repo supports git LFS
         self.run_program(['git', 'lfs', 'install', '--local'], cwd=gitdir)
 
-        for src in new_files:
-            self.run_program(["%s.raw-file-importer" % lorry_path, new_files[src], src], cwd=gitdir)
+        for subpath, raw_file in new_files:
+            self.run_program(["%s.raw-file-importer" % lorry_path, raw_file, subpath], cwd=gitdir)
 
     def gitify_archive(self, archive_type, project_name, dirname, gitdir, spec):
         assert archive_type in ['zip', 'gzip', 'tar']