Handle errors while a project is being deleted asynchronously.

1. Rescue all errors that `Projects::DestroyService` might throw, to prevent the
   worker from leaving things in an inconsistent state

2. Unmark the project as `pending_delete`

3. Add a `delete_error` text column to `projects`, and save the error message in
   there, to be shown to the project masters/owners.

2 files changed, 8 insertions, 4 deletions

diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index e2b2660ea71..682407ac896 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -26,9 +26,6 @@ module Projects
       Projects::UnlinkForkService.new(project, current_user).execute
 
       Project.transaction do
-        project.team.truncate
-        project.destroy!
-
         unless remove_legacy_registry_tags
           raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.')
         end
@@ -40,10 +37,14 @@ module Projects
         unless remove_repository(wiki_path)
           raise_error('Failed to remove wiki repository. Please try again or contact administrator.')
         end
+
+        project.team.truncate
+        project.destroy!
       end
 
-      log_info("Project \"#{project.path_with_namespace}\" was removed")
       system_hook_service.execute_hooks_for(project, :destroy)
+
+      log_info("Project \"#{project.path_with_namespace}\" was removed")
       true
     end
 
diff --git a/app/workers/project_destroy_worker.rb b/app/workers/project_destroy_worker.rb
index b462327490e..482e1e38cd1 100644
--- a/app/workers/project_destroy_worker.rb
+++ b/app/workers/project_destroy_worker.rb
@@ -12,5 +12,8 @@ class ProjectDestroyWorker
     user = User.find(user_id)
 
     ::Projects::DestroyService.new(project, user, params.symbolize_keys).execute
+  rescue StandardError => error
+    project.assign_attributes(delete_error: error.message, pending_delete: false)
+    project.save!(validate: false)
   end
 end