Merge branch '23223-group-deletion-race-condition' into 'master'

Remove race condition while deleting groups

## What does this MR do?

The intended flow during a group deletion is:

```
  Soft-delete group (sync) -> Delete group projects (async) -> Hard-delete group (async)
```

The soft-delete was run in a transaction, which was committed only after
the async job (for hard-deletion) was kicked off. There was a race
condition here - the soft-delete transaction could complete _after_ the
hard delete completed, leaving a soft-deleted record in the database.

This MR removes this race condition. There is no need to run the
soft-delete in a transaction. The soft-delete completes before the async
job is kicked off.

This MR also adds a migration to delete all existing (soft-deleted) groups left in an inconsistent state
due to this bug.

- Closes #23223 
- EE merge request: gitlab-org/gitlab-ee!886

See merge request !7528

2 files changed, 49 insertions, 0 deletions

diff --git a/spec/services/destroy_group_service_spec.rb b/spec/services/destroy_group_service_spec.rb
index da724643604..538e85cdc89 100644
--- a/spec/services/destroy_group_service_spec.rb
+++ b/spec/services/destroy_group_service_spec.rb
@@ -1,6 +1,8 @@
 require 'spec_helper'
 
 describe DestroyGroupService, services: true do
+  include DatabaseConnectionHelpers
+
   let!(:user) { create(:user) }
   let!(:group) { create(:group) }
   let!(:project) { create(:project, namespace: group) }
@@ -50,6 +52,44 @@ describe DestroyGroupService, services: true do
 
   describe 'asynchronous delete' do
     it_behaves_like 'group destruction', true
+
+    context 'potential race conditions' do
+      context "when the `GroupDestroyWorker` task runs immediately" do
+        it "deletes the group" do
+          # Commit the contents of this spec's transaction so far
+          # so subsequent db connections can see it.
+          #
+          # DO NOT REMOVE THIS LINE, even if you see a WARNING with "No
+          # transaction is currently in progress". Without this, this
+          # spec will always be green, since the group created in setup
+          # cannot be seen by any other connections / threads in this spec.
+          Group.connection.commit_db_transaction
+
+          group_record = run_with_new_database_connection do |conn|
+            conn.execute("SELECT * FROM namespaces WHERE id = #{group.id}").first
+          end
+
+          expect(group_record).not_to be_nil
+
+          # Execute the contents of `GroupDestroyWorker` in a separate thread, to
+          # simulate data manipulation by the Sidekiq worker (different database
+          # connection / transaction).
+          expect(GroupDestroyWorker).to receive(:perform_async).and_wrap_original do |m, group_id, user_id|
+            Thread.new { m[group_id, user_id] }.join(5)
+          end
+
+          # Kick off the initial group destroy in a new thread, so that
+          # it doesn't share this spec's database transaction.
+          Thread.new { DestroyGroupService.new(group, user).async_execute }.join(5)
+
+          group_record = run_with_new_database_connection do |conn|
+            conn.execute("SELECT * FROM namespaces WHERE id = #{group.id}").first
+          end
+
+          expect(group_record).to be_nil
+        end
+      end
+    end
   end
 
   describe 'synchronous delete' do
diff --git a/spec/support/database_connection_helpers.rb b/spec/support/database_connection_helpers.rb
new file mode 100644
index 00000000000..763329499f0
--- /dev/null
+++ b/spec/support/database_connection_helpers.rb
@@ -0,0 +1,9 @@
+module DatabaseConnectionHelpers
+  def run_with_new_database_connection
+    pool = ActiveRecord::Base.connection_pool
+    conn = pool.checkout
+    yield conn
+  ensure
+    pool.checkin(conn)
+  end
+end