Remove persistent configuration downstream

Checks whether downstream files are now redundant according to current
.lorry file, using `git worktree`. If so, their paths are deleted
downstream.

Also checks whether redundant files are not needed locally, ie the files
haven't just been moved to a new path. If so, they're deleted.

Resolves #30

1 files changed, 96 insertions, 21 deletions

diff --git a/lorry b/lorry
index 935f962..5fa0569 100755
--- a/lorry
+++ b/lorry
@@ -201,7 +201,7 @@ class Lorry(cliapp.Application):
         )
         self.settings.string(
             ["mirror-base-url-push"],
-            "base URL to use for pushing to the mirror " "server",
+            "base URL to use for pushing to the mirror server",
             metavar="URL",
         )
         self.settings.string(
@@ -212,7 +212,7 @@ class Lorry(cliapp.Application):
         )
         self.settings.boolean(
             ["pull-only"],
-            "only pull from upstreams, do not push to " "the mirror server",
+            "only pull from upstreams, do not push to the mirror server",
         )
         self.settings.boolean(["verbose", "v"], "report what is going on to stdout")
         self.settings.boolean(
@@ -262,7 +262,7 @@ class Lorry(cliapp.Application):
             default=False,
         )
         self.settings.string_list(
-            ["push-option"], "option for 'git push' to pass to the " "remote server"
+            ["push-option"], "option for 'git push' to pass to the remote server"
         )
         self.settings.string(
             ["bazaar-command"],
@@ -482,7 +482,7 @@ class Lorry(cliapp.Application):
         if os.path.exists(old_repo):
             new_repo = os.path.join(dirname, "git-a")
             if os.path.exists(new_repo):
-                msg = "Found both old %s and new %s directories; " "not migrating\n" % (
+                msg = "Found both old %s and new %s directories; not migrating\n" % (
                     old_repo,
                     new_repo,
                 )
@@ -904,16 +904,45 @@ class Lorry(cliapp.Application):
             branch=raw_file_branch
         )
         self.ensure_gitdir(gitdir)
-        # Fetch the files
+
+        # Ensure the repo is up-to-date
+        pullurl = "%s/%s.git" % (self.settings["mirror-base-url-push"], project_name)
+        try:
+            self.run_program(["git", "fetch", pullurl, raw_file_refspecs], cwd=gitdir)
+        except Exception:
+            # TODO: Be more specific about which exceptions are fine
+            self.progress("Failed to fetch from URL: %s" % pullurl)
+
+        # Ensure the repo supports git LFS
+        self.run_program(["git", "lfs", "install", "--local"], cwd=gitdir)
+
+        try:
+            # List of all files in preexisting downstream repo
+            old_files = (
+                self.run_program(
+                    ["git", "ls-tree", "-r", "HEAD", "--name-only"], cwd=gitdir
+                )
+                .strip()
+                .splitlines()
+            )
+        except Exception:
+            old_files = []
+
+        # Fetch the files specified in .lorry file
         new_files = []
+        desired_files = [".gitattributes"]
         for src in spec["urls"]:
             url = src["url"]
             url_path = urllib.parse.urlparse(url)[2]
             basename = os.path.basename(url_path)
             file_dest = os.path.join(dirname, basename)
             self.progress(".. checking if we need to fetch %s" % basename)
+            repo_subdir = src.get("destination", ".")
+            repo_dest = os.path.relpath(os.path.join(repo_subdir, basename))
+            desired_files.append(repo_dest)
+
             if file_missing_or_empty(file_dest):
-                new_files.append((src.get("destination", "."), file_dest))
+                new_files.append((repo_subdir, file_dest))
                 self.progress(".. attempting to fetch %s" % basename)
                 try:
                     with open(file_dest, "wb") as raw_file, self.urlopen(
@@ -935,28 +964,74 @@ class Lorry(cliapp.Application):
                     if os.path.exists(file_dest):
                         os.unlink(file_dest)
                     raise
+            elif repo_dest not in old_files:
+                new_files.append((repo_subdir, file_dest))
+                self.progress("..path has changed for %s" % basename)
             else:
-                self.progress("nothing to do for %s" % basename)
-
-        if not len(new_files):
-            self.progress(".. no need to run, nothing to do")
-            return
+                self.progress("no need to import %s" % basename)
 
-        # Ensure the repo is up-to-date
-        pullurl = "%s/%s.git" % (self.settings["mirror-base-url-push"], project_name)
-        try:
-            self.run_program(["git", "fetch", pullurl, raw_file_refspecs], cwd=gitdir)
-        except Exception:
-            # TODO: Be more specific about which exceptions are fine
-            pass
-
-        # Ensure the repo supports git LFS
-        self.run_program(["git", "lfs", "install", "--local"], cwd=gitdir)
+        if len(new_files) == 0:
+            self.progress(".. no need to run importer")
 
+        # Import files to bare local git repo
         for subpath, raw_file in new_files:
             self.run_program(
                 ["%s.raw-file-importer" % lorry_path, raw_file, subpath], cwd=gitdir
             )
+        # Set user info for commits
+        self.run_program(
+            ["git", "config", "user.name", '"Lorry Raw File Importer"'], cwd=gitdir
+        )
+        self.run_program(
+            ["git", "config", "user.email", '"lorry-raw-file-importer@lorry"'],
+            cwd=gitdir,
+        )
+
+        # Remove repo file paths of worktree that aren't
+        # included included in .lorry file
+        # If old worktree exists, delete it
+        gitdir_prefix = os.path.dirname(gitdir)
+        worktree = os.path.join(gitdir_prefix, "raw-file-worktree")
+
+        if os.path.exists(worktree):
+            shutil.rmtree(worktree)
+        self.run_program(["git", "worktree", "prune"], cwd=gitdir)
+
+        # Create worktree from preexisting downstream repo
+        self.run_program(
+            ["git", "worktree", "add", worktree, "--checkout", raw_file_branch],
+            cwd=gitdir,
+        )
+
+        unexpected_files = [
+            old_file for old_file in old_files if old_file not in desired_files
+        ]
+        for file_path in unexpected_files:
+            self.progress("Found unexpected file: %s" % file_path)
+            # Delete file path from git repo
+            self.run_program(
+                ["git", "rm", "-f", file_path],
+                cwd=worktree,
+            )
+            # Delete file locally if not needed whatsoever
+            file_name = os.path.basename(file_path)
+            if file_name not in [
+                os.path.basename(file_path) for file_path in desired_files
+            ]:
+                file_abs_path_local = os.path.join(gitdir_prefix, file_name)
+                os.remove(file_abs_path_local)
+                self.progress("Deleted unwanted file locally: %s" % file_name)
+        # Commit changes
+        try:
+            self.progress(".. commiting deletions of unwanted files")
+            self.run_program(
+                ["git", "commit", "-m", "Remove redundant file paths."], cwd=worktree
+            )
+        except Exception:
+            self.progress("Couldn't commit deletions. Perhaps there weren't any?")
+        # Remove worktree (in case files are large)
+        shutil.rmtree(worktree)
+        self.run_program(["git", "worktree", "prune"], cwd=gitdir)
 
     def gitify_archive(self, archive_type, project_name, dirname, gitdir, spec):
         assert archive_type in ["zip", "gzip", "tar"]