Merge branch 'sh-ignore-git-errors-delete-project' into 'master'

Ignore Gitaly errors if cache flushing fails on project destruction

See merge request gitlab-org/gitlab-ce!31164

3 files changed, 32 insertions, 3 deletions

diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index b805a7f1211..a1279bfb3a3 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -210,11 +210,20 @@ module Projects
     end
 
     def flush_caches(project)
-      project.repository.before_delete
+      ignore_git_errors(repo_path) { project.repository.before_delete }
 
-      Repository.new(wiki_path, project, disk_path: repo_path).before_delete
+      ignore_git_errors(wiki_path) { Repository.new(wiki_path, project, disk_path: repo_path).before_delete }
 
       Projects::ForksCountService.new(project).delete_cache
     end
+
+    # If we get a Gitaly error, the repository may be corrupted. We can
+    # ignore these errors since we're going to trash the repositories
+    # anyway.
+    def ignore_git_errors(disk_path, &block)
+      yield
+    rescue Gitlab::Git::CommandError => e
+      Gitlab::GitLogger.warn(class: self.class.name, project_id: project.id, disk_path: disk_path, message: e.to_s)
+    end
   end
 end
diff --git a/changelogs/unreleased/sh-ignore-git-errors-delete-project.yml b/changelogs/unreleased/sh-ignore-git-errors-delete-project.yml
new file mode 100644
index 00000000000..f5e2147f00e
--- /dev/null
+++ b/changelogs/unreleased/sh-ignore-git-errors-delete-project.yml
@@ -0,0 +1,5 @@
+---
+title: Ignore Gitaly errors if cache flushing fails on project destruction
+merge_request: 31164
+author:
+type: fixed
diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb
index 3af7ee3ad50..e436af77ed4 100644
--- a/spec/services/projects/destroy_service_spec.rb
+++ b/spec/services/projects/destroy_service_spec.rb
@@ -121,7 +121,22 @@ describe Projects::DestroyService do
     it { expect(Dir.exist?(remove_path)).to be_truthy }
   end
 
-  context 'when flushing caches fail' do
+  context 'when flushing caches fail due to Git errors' do
+    before do
+      allow(project.repository).to receive(:before_delete).and_raise(::Gitlab::Git::CommandError)
+      allow(Gitlab::GitLogger).to receive(:warn).with(
+        class: described_class.name,
+        project_id: project.id,
+        disk_path: project.disk_path,
+        message: 'Gitlab::Git::CommandError').and_call_original
+
+      perform_enqueued_jobs { destroy_project(project, user, {}) }
+    end
+
+    it_behaves_like 'deleting the project'
+  end
+
+  context 'when flushing caches fail due to Redis' do
     before do
       new_user = create(:user)
       project.team.add_user(new_user, Gitlab::Access::DEVELOPER)