Look up upstream commits once before queuing ProcessCommitWorkers

Instead of checking if a commit already exists in the upstream project
in its ProcessCommitWorker and bailing out if it does, we check the
existence of all commits in bulk in Git::BranchHooksService, so that we
can skip scheduling ProcessCommitWorker jobs for those commits
that already exist upstream entirely.

2 files changed, 60 insertions, 50 deletions

diff --git a/spec/services/git/branch_hooks_service_spec.rb b/spec/services/git/branch_hooks_service_spec.rb
index 23be400059e..41180402759 100644
--- a/spec/services/git/branch_hooks_service_spec.rb
+++ b/spec/services/git/branch_hooks_service_spec.rb
@@ -4,6 +4,7 @@ require 'spec_helper'
 
 describe Git::BranchHooksService do
   include RepoHelpers
+  include ProjectForksHelper
 
   let(:project) { create(:project, :repository) }
   let(:user) { project.creator }
@@ -266,10 +267,10 @@ describe Git::BranchHooksService do
   end
 
   describe 'Processing commit messages' do
-    # Create 4 commits, 2 of which have references. Limiting to 2 commits, we
-    # expect to see one commit message processor enqueued.
-    let(:commit_ids) do
-      Array.new(4) do |i|
+    # Create 6 commits, 3 of which have references. Limiting to 4 commits, we
+    # expect to see two commit message processors enqueued.
+    let!(:commit_ids) do
+      Array.new(6) do |i|
         message = "Issue #{'#' if i.even?}#{i}"
         project.repository.update_file(
           user, 'README.md', '', message: message, branch_name: branch
@@ -277,18 +278,18 @@ describe Git::BranchHooksService do
       end
     end
 
-    let(:oldrev) { commit_ids.first }
+    let(:oldrev) { project.commit(commit_ids.first).parent_id }
     let(:newrev) { commit_ids.last }
 
     before do
-      stub_const("::Git::BaseHooksService::PROCESS_COMMIT_LIMIT", 2)
+      stub_const("::Git::BaseHooksService::PROCESS_COMMIT_LIMIT", 4)
     end
 
     context 'creating the default branch' do
       let(:oldrev) { Gitlab::Git::BLANK_SHA }
 
       it 'processes a limited number of commit messages' do
-        expect(ProcessCommitWorker).to receive(:perform_async).once
+        expect(ProcessCommitWorker).to receive(:perform_async).twice
 
         service.execute
       end
@@ -296,7 +297,7 @@ describe Git::BranchHooksService do
 
     context 'updating the default branch' do
       it 'processes a limited number of commit messages' do
-        expect(ProcessCommitWorker).to receive(:perform_async).once
+        expect(ProcessCommitWorker).to receive(:perform_async).twice
 
         service.execute
       end
@@ -317,7 +318,7 @@ describe Git::BranchHooksService do
       let(:oldrev) { Gitlab::Git::BLANK_SHA }
 
       it 'processes a limited number of commit messages' do
-        expect(ProcessCommitWorker).to receive(:perform_async).once
+        expect(ProcessCommitWorker).to receive(:perform_async).twice
 
         service.execute
       end
@@ -327,7 +328,7 @@ describe Git::BranchHooksService do
       let(:branch) { 'fix' }
 
       it 'processes a limited number of commit messages' do
-        expect(ProcessCommitWorker).to receive(:perform_async).once
+        expect(ProcessCommitWorker).to receive(:perform_async).twice
 
         service.execute
       end
@@ -343,6 +344,55 @@ describe Git::BranchHooksService do
         service.execute
       end
     end
+
+    context 'when the project is forked' do
+      let(:upstream_project) { project }
+      let(:forked_project) { fork_project(upstream_project, user, repository: true) }
+
+      let!(:forked_service) do
+        described_class.new(forked_project, user, oldrev: oldrev, newrev: newrev, ref: ref)
+      end
+
+      context 'when commits already exists in the upstream project' do
+        it 'does not process commit messages' do
+          expect(ProcessCommitWorker).not_to receive(:perform_async)
+
+          forked_service.execute
+        end
+      end
+
+      context 'when a commit does not exist in the upstream repo' do
+        # On top of the existing 6 commits, 3 of which have references,
+        # create 2 more, 1 of which has a reference. Limiting to 4 commits, we
+        # expect to see one commit message processor enqueued.
+        let!(:forked_commit_ids) do
+          Array.new(2) do |i|
+            message = "Issue #{'#' if i.even?}#{i}"
+            forked_project.repository.update_file(
+              user, 'README.md', '', message: message, branch_name: branch
+            )
+          end
+        end
+
+        let(:newrev) { forked_commit_ids.last }
+
+        it 'processes the commit message' do
+          expect(ProcessCommitWorker).to receive(:perform_async).once
+
+          forked_service.execute
+        end
+      end
+
+      context 'when the upstream project no longer exists' do
+        it 'processes the commit messages' do
+          upstream_project.destroy!
+
+          expect(ProcessCommitWorker).to receive(:perform_async).twice
+
+          forked_service.execute
+        end
+      end
+    end
   end
 
   describe 'New branch detection' do
diff --git a/spec/workers/process_commit_worker_spec.rb b/spec/workers/process_commit_worker_spec.rb
index 47bac63511e..eb1d3c364ac 100644
--- a/spec/workers/process_commit_worker_spec.rb
+++ b/spec/workers/process_commit_worker_spec.rb
@@ -3,8 +3,6 @@
 require 'spec_helper'
 
 describe ProcessCommitWorker do
-  include ProjectForksHelper
-
   let(:worker) { described_class.new }
   let(:user) { create(:user) }
   let(:project) { create(:project, :public, :repository) }
@@ -35,44 +33,6 @@ describe ProcessCommitWorker do
 
       worker.perform(project.id, user.id, commit.to_hash)
     end
-
-    context 'when the project is forked' do
-      context 'when commit already exists in the upstream project' do
-        it 'does not process the commit message' do
-          forked = fork_project(project, user, repository: true)
-
-          expect(worker).not_to receive(:process_commit_message)
-
-          worker.perform(forked.id, user.id, forked.commit.to_hash)
-        end
-      end
-
-      context 'when the commit does not exist in the upstream project' do
-        it 'processes the commit message' do
-          empty_project = create(:project, :public)
-          forked = fork_project(empty_project, user, repository: true)
-
-          TestEnv.copy_repo(forked,
-                            bare_repo: TestEnv.factory_repo_path_bare,
-                            refs: TestEnv::BRANCH_SHA)
-
-          expect(worker).to receive(:process_commit_message)
-
-          worker.perform(forked.id, user.id, forked.commit.to_hash)
-        end
-      end
-
-      context 'when the upstream project no longer exists' do
-        it 'processes the commit message' do
-          forked = fork_project(project, user, repository: true)
-          project.destroy!
-
-          expect(worker).to receive(:process_commit_message)
-
-          worker.perform(forked.id, user.id, forked.commit.to_hash)
-        end
-      end
-    end
   end
 
   describe '#process_commit_message' do