From b528aa6f81e747a2670a138ba5710f0755a81355 Mon Sep 17 00:00:00 2001 From: Chris Shepherd Date: Thu, 10 Feb 2022 12:35:50 +0000 Subject: Remove persistent configuration downstream Checks whether downstream files are now redundant according to current .lorry file, using `git worktree`. If so, their paths are deleted downstream. Also checks whether redundant files are not needed locally, ie the files haven't just been moved to a new path. If so, they're deleted. Resolves #30 --- lorry | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 96 insertions(+), 21 deletions(-) (limited to 'lorry') diff --git a/lorry b/lorry index 935f962..5fa0569 100755 --- a/lorry +++ b/lorry @@ -201,7 +201,7 @@ class Lorry(cliapp.Application): ) self.settings.string( ["mirror-base-url-push"], - "base URL to use for pushing to the mirror " "server", + "base URL to use for pushing to the mirror server", metavar="URL", ) self.settings.string( @@ -212,7 +212,7 @@ class Lorry(cliapp.Application): ) self.settings.boolean( ["pull-only"], - "only pull from upstreams, do not push to " "the mirror server", + "only pull from upstreams, do not push to the mirror server", ) self.settings.boolean(["verbose", "v"], "report what is going on to stdout") self.settings.boolean( @@ -262,7 +262,7 @@ class Lorry(cliapp.Application): default=False, ) self.settings.string_list( - ["push-option"], "option for 'git push' to pass to the " "remote server" + ["push-option"], "option for 'git push' to pass to the remote server" ) self.settings.string( ["bazaar-command"], @@ -482,7 +482,7 @@ class Lorry(cliapp.Application): if os.path.exists(old_repo): new_repo = os.path.join(dirname, "git-a") if os.path.exists(new_repo): - msg = "Found both old %s and new %s directories; " "not migrating\n" % ( + msg = "Found both old %s and new %s directories; not migrating\n" % ( old_repo, new_repo, ) @@ -904,16 +904,45 @@ class Lorry(cliapp.Application): branch=raw_file_branch ) self.ensure_gitdir(gitdir) - # Fetch the files + + # Ensure the repo is up-to-date + pullurl = "%s/%s.git" % (self.settings["mirror-base-url-push"], project_name) + try: + self.run_program(["git", "fetch", pullurl, raw_file_refspecs], cwd=gitdir) + except Exception: + # TODO: Be more specific about which exceptions are fine + self.progress("Failed to fetch from URL: %s" % pullurl) + + # Ensure the repo supports git LFS + self.run_program(["git", "lfs", "install", "--local"], cwd=gitdir) + + try: + # List of all files in preexisting downstream repo + old_files = ( + self.run_program( + ["git", "ls-tree", "-r", "HEAD", "--name-only"], cwd=gitdir + ) + .strip() + .splitlines() + ) + except Exception: + old_files = [] + + # Fetch the files specified in .lorry file new_files = [] + desired_files = [".gitattributes"] for src in spec["urls"]: url = src["url"] url_path = urllib.parse.urlparse(url)[2] basename = os.path.basename(url_path) file_dest = os.path.join(dirname, basename) self.progress(".. checking if we need to fetch %s" % basename) + repo_subdir = src.get("destination", ".") + repo_dest = os.path.relpath(os.path.join(repo_subdir, basename)) + desired_files.append(repo_dest) + if file_missing_or_empty(file_dest): - new_files.append((src.get("destination", "."), file_dest)) + new_files.append((repo_subdir, file_dest)) self.progress(".. attempting to fetch %s" % basename) try: with open(file_dest, "wb") as raw_file, self.urlopen( @@ -935,28 +964,74 @@ class Lorry(cliapp.Application): if os.path.exists(file_dest): os.unlink(file_dest) raise + elif repo_dest not in old_files: + new_files.append((repo_subdir, file_dest)) + self.progress("..path has changed for %s" % basename) else: - self.progress("nothing to do for %s" % basename) - - if not len(new_files): - self.progress(".. no need to run, nothing to do") - return + self.progress("no need to import %s" % basename) - # Ensure the repo is up-to-date - pullurl = "%s/%s.git" % (self.settings["mirror-base-url-push"], project_name) - try: - self.run_program(["git", "fetch", pullurl, raw_file_refspecs], cwd=gitdir) - except Exception: - # TODO: Be more specific about which exceptions are fine - pass - - # Ensure the repo supports git LFS - self.run_program(["git", "lfs", "install", "--local"], cwd=gitdir) + if len(new_files) == 0: + self.progress(".. no need to run importer") + # Import files to bare local git repo for subpath, raw_file in new_files: self.run_program( ["%s.raw-file-importer" % lorry_path, raw_file, subpath], cwd=gitdir ) + # Set user info for commits + self.run_program( + ["git", "config", "user.name", '"Lorry Raw File Importer"'], cwd=gitdir + ) + self.run_program( + ["git", "config", "user.email", '"lorry-raw-file-importer@lorry"'], + cwd=gitdir, + ) + + # Remove repo file paths of worktree that aren't + # included included in .lorry file + # If old worktree exists, delete it + gitdir_prefix = os.path.dirname(gitdir) + worktree = os.path.join(gitdir_prefix, "raw-file-worktree") + + if os.path.exists(worktree): + shutil.rmtree(worktree) + self.run_program(["git", "worktree", "prune"], cwd=gitdir) + + # Create worktree from preexisting downstream repo + self.run_program( + ["git", "worktree", "add", worktree, "--checkout", raw_file_branch], + cwd=gitdir, + ) + + unexpected_files = [ + old_file for old_file in old_files if old_file not in desired_files + ] + for file_path in unexpected_files: + self.progress("Found unexpected file: %s" % file_path) + # Delete file path from git repo + self.run_program( + ["git", "rm", "-f", file_path], + cwd=worktree, + ) + # Delete file locally if not needed whatsoever + file_name = os.path.basename(file_path) + if file_name not in [ + os.path.basename(file_path) for file_path in desired_files + ]: + file_abs_path_local = os.path.join(gitdir_prefix, file_name) + os.remove(file_abs_path_local) + self.progress("Deleted unwanted file locally: %s" % file_name) + # Commit changes + try: + self.progress(".. commiting deletions of unwanted files") + self.run_program( + ["git", "commit", "-m", "Remove redundant file paths."], cwd=worktree + ) + except Exception: + self.progress("Couldn't commit deletions. Perhaps there weren't any?") + # Remove worktree (in case files are large) + shutil.rmtree(worktree) + self.run_program(["git", "worktree", "prune"], cwd=gitdir) def gitify_archive(self, archive_type, project_name, dirname, gitdir, spec): assert archive_type in ["zip", "gzip", "tar"] -- cgit v1.2.1