Merge branch '29289-project-destroy-clean-up-after-failure' into 'master'

Handle errors while a project is being deleted asynchronously.

Closes #29289

See merge request !11088

6 files changed, 69 insertions, 38 deletions

diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index e2b2660ea71..f6e8b6655f2 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -15,40 +15,48 @@ module Projects
     def execute
       return false unless can?(current_user, :remove_project, project)
 
-      repo_path = project.path_with_namespace
-      wiki_path = repo_path + '.wiki'
-
       # Flush the cache for both repositories. This has to be done _before_
       # removing the physical repositories as some expiration code depends on
       # Git data (e.g. a list of branch names).
-      flush_caches(project, wiki_path)
+      flush_caches(project)
 
       Projects::UnlinkForkService.new(project, current_user).execute
 
-      Project.transaction do
-        project.team.truncate
-        project.destroy!
-
-        unless remove_legacy_registry_tags
-          raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.')
-        end
-
-        unless remove_repository(repo_path)
-          raise_error('Failed to remove project repository. Please try again or contact administrator.')
-        end
+      attempt_destroy_transaction(project)
 
-        unless remove_repository(wiki_path)
-          raise_error('Failed to remove wiki repository. Please try again or contact administrator.')
-        end
-      end
-
-      log_info("Project \"#{project.path_with_namespace}\" was removed")
       system_hook_service.execute_hooks_for(project, :destroy)
+      log_info("Project \"#{project.full_path}\" was removed")
+
       true
+    rescue => error
+      attempt_rollback(project, error.message)
+      false
+    rescue Exception => error # rubocop:disable Lint/RescueException
+      # Project.transaction can raise Exception
+      attempt_rollback(project, error.message)
+      raise
     end
 
     private
 
+    def repo_path
+      project.path_with_namespace
+    end
+
+    def wiki_path
+      repo_path + '.wiki'
+    end
+
+    def trash_repositories!
+      unless remove_repository(repo_path)
+        raise_error('Failed to remove project repository. Please try again or contact administrator.')
+      end
+
+      unless remove_repository(wiki_path)
+        raise_error('Failed to remove wiki repository. Please try again or contact administrator.')
+      end
+    end
+
     def remove_repository(path)
       # Skip repository removal. We use this flag when remove user or group
       return true if params[:skip_repo] == true
@@ -70,6 +78,26 @@ module Projects
       end
     end
 
+    def attempt_rollback(project, message)
+      return unless project
+
+      project.update_attributes(delete_error: message, pending_delete: false)
+      log_error("Deletion failed on #{project.full_path} with the following message: #{message}")
+    end
+
+    def attempt_destroy_transaction(project)
+      Project.transaction do
+        unless remove_legacy_registry_tags
+          raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.')
+        end
+
+        trash_repositories!
+
+        project.team.truncate
+        project.destroy!
+      end
+    end
+
     ##
     # This method makes sure that we correctly remove registry tags
     # for legacy image repository (when repository path equals project path).
@@ -96,7 +124,7 @@ module Projects
       "#{path}+#{project.id}#{DELETED_FLAG}"
     end
 
-    def flush_caches(project, wiki_path)
+    def flush_caches(project)
       project.repository.before_delete
 
       Repository.new(wiki_path, project).before_delete
diff --git a/app/views/projects/_deletion_failed.html.haml b/app/views/projects/_deletion_failed.html.haml
new file mode 100644
index 00000000000..4f3698f91e6
--- /dev/null
+++ b/app/views/projects/_deletion_failed.html.haml
@@ -0,0 +1,6 @@
+- project = local_assigns.fetch(:project)
+- return unless project.delete_error.present?
+
+.project-deletion-failed-message.alert.alert-warning
+  This project was scheduled for deletion, but failed with the following message:
+  = project.delete_error
diff --git a/app/views/projects/_flash_messages.html.haml b/app/views/projects/_flash_messages.html.haml
new file mode 100644
index 00000000000..f47d84ef755
--- /dev/null
+++ b/app/views/projects/_flash_messages.html.haml
@@ -0,0 +1,8 @@
+- project = local_assigns.fetch(:project)
+- flash_message_container = show_new_nav? ? :new_global_flash : :flash_message
+
+= content_for flash_message_container do
+  = render partial: 'deletion_failed', locals: { project: project }
+  - if current_user && can?(current_user, :download_code, project)
+    = render 'shared/no_ssh'
+    = render 'shared/no_password'
diff --git a/app/views/projects/empty.html.haml b/app/views/projects/empty.html.haml
index 0f132a68ce1..d17709380d5 100644
--- a/app/views/projects/empty.html.haml
+++ b/app/views/projects/empty.html.haml
@@ -1,10 +1,6 @@
 - @no_container = true
-- flash_message_container = show_new_nav? ? :new_global_flash : :flash_message
 
-= content_for flash_message_container do
-  - if current_user && can?(current_user, :download_code, @project)
-    = render 'shared/no_ssh'
-    = render 'shared/no_password'
+= render partial: 'flash_messages', locals: { project: @project }
 
 = render "projects/head"
 = render "home_panel"
diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml
index 49d0a6828fe..a9b39cedb1d 100644
--- a/app/views/projects/show.html.haml
+++ b/app/views/projects/show.html.haml
@@ -1,15 +1,11 @@
 - @no_container = true
 - breadcrumb_title "Project"
 - @content_class = "limit-container-width" unless fluid_layout
-- flash_message_container = show_new_nav? ? :new_global_flash : :flash_message
 
 = content_for :meta_tags do
   = auto_discovery_link_tag(:atom, project_path(@project, rss_url_options), title: "#{@project.name} activity")
 
-= content_for flash_message_container do
-  - if current_user && can?(current_user, :download_code, @project)
-    = render 'shared/no_ssh'
-    = render 'shared/no_password'
+= render partial: 'flash_messages', locals: { project: @project }
 
 = render "projects/head"
 = render "projects/last_push"
diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb
index b462327490e..a9188b78460 100644
--- a/app/workers/project_destroy_worker.rb
+++ b/app/workers/project_destroy_worker.rb
@@ -3,14 +3,11 @@ class ProjectDestroyWorker
   include DedicatedSidekiqQueue
 
   def perform(project_id, user_id, params)
-    begin
-      project = Project.unscoped.find(project_id)
-    rescue ActiveRecord::RecordNotFound
-      return
-    end
-
+    project = Project.find(project_id)
     user = User.find(user_id)
 
     ::Projects::DestroyService.new(project, user, params.symbolize_keys).execute
+  rescue ActiveRecord::RecordNotFound => error
+    logger.error("Failed to delete project (#{project_id}): #{error.message}")
   end
 end