Make pipeline processing specs more consistent

1 files changed, 316 insertions, 278 deletions

diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index d2cb1f5c0a0..2cceca40a46 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Ci::ProcessPipelineService, :services do
+describe Ci::ProcessPipelineService, '#execute', :services do
   let(:user) { create(:user) }
   let(:project) { create(:empty_project) }
 
@@ -12,363 +12,389 @@ describe Ci::ProcessPipelineService, :services do
     project.add_developer(user)
   end
 
-  describe '#execute' do
-    context 'start queuing next builds' do
-      before do
-        create_build('linux', stage_idx: 0)
-        create_build('mac', stage_idx: 0)
-        create_build('rspec', stage_idx: 1)
-        create_build('rubocop', stage_idx: 1)
-        create_build('deploy', stage_idx: 2)
-      end
+  context 'start queuing next builds' do
+    before do
+      create_build('linux', stage_idx: 0)
+      create_build('mac', stage_idx: 0)
+      create_build('rspec', stage_idx: 1)
+      create_build('rubocop', stage_idx: 1)
+      create_build('deploy', stage_idx: 2)
+    end
 
-      it 'processes a pipeline' do
-        expect(process_pipeline).to be_truthy
-        succeed_pending
-        expect(builds.success.count).to eq(2)
+    it 'processes a pipeline' do
+      expect(process_pipeline).to be_truthy
 
-        expect(process_pipeline).to be_truthy
-        succeed_pending
-        expect(builds.success.count).to eq(4)
+      succeed_pending
 
-        expect(process_pipeline).to be_truthy
-        succeed_pending
-        expect(builds.success.count).to eq(5)
+      expect(builds.success.count).to eq(2)
+      expect(process_pipeline).to be_truthy
 
-        expect(process_pipeline).to be_falsey
-      end
+      succeed_pending
 
-      it 'does not process pipeline if existing stage is running' do
-        expect(process_pipeline).to be_truthy
-        expect(builds.pending.count).to eq(2)
+      expect(builds.success.count).to eq(4)
+      expect(process_pipeline).to be_truthy
 
-        expect(process_pipeline).to be_falsey
-        expect(builds.pending.count).to eq(2)
-      end
+      succeed_pending
+
+      expect(builds.success.count).to eq(5)
+      expect(process_pipeline).to be_falsey
     end
 
-    context 'custom stage with first job allowed to fail' do
-      before do
-        create_build('clean_job', stage_idx: 0, allow_failure: true)
-        create_build('test_job', stage_idx: 1, allow_failure: true)
-      end
+    it 'does not process pipeline if existing stage is running' do
+      expect(process_pipeline).to be_truthy
+      expect(builds.pending.count).to eq(2)
 
-      it 'automatically triggers a next stage when build finishes' do
+      expect(process_pipeline).to be_falsey
+      expect(builds.pending.count).to eq(2)
+    end
+  end
+
+  context 'custom stage with first job allowed to fail' do
+    before do
+      create_build('clean_job', stage_idx: 0, allow_failure: true)
+      create_build('test_job', stage_idx: 1, allow_failure: true)
+    end
+
+    it 'automatically triggers a next stage when build finishes' do
+      expect(process_pipeline).to be_truthy
+      expect(builds_statuses).to eq ['pending']
+
+      fail_running_or_pending
+
+      expect(builds_statuses).to eq ['failed', 'pending']
+    end
+  end
+
+  context 'properly creates builds when optional actions are defined' do
+    before do
+      create_build('build', stage_idx: 0)
+      create_build('test', stage_idx: 1)
+      create_build('test_failure', stage_idx: 2, when: 'on_failure')
+      create_build('deploy', stage_idx: 3)
+      create_build('production', stage_idx: 3, when: 'manual', allow_failure: true)
+      create_build('cleanup', stage_idx: 4, when: 'always')
+      create_build('clear cache', stage_idx: 4, when: 'manual', allow_failure: true)
+    end
+
+    context 'when builds are successful' do
+      it 'properly creates builds' do
         expect(process_pipeline).to be_truthy
-        expect(builds.pluck(:status)).to contain_exactly('pending')
+        expect(builds_names).to eq ['build']
+        expect(builds_statuses).to eq ['pending']
+
+        succeed_running_or_pending
+
+        expect(builds_names).to eq ['build', 'test']
+        expect(builds_statuses).to eq ['success', 'pending']
+
+        succeed_running_or_pending
 
-        pipeline.builds.running_or_pending.each(&:drop)
-        expect(builds.pluck(:status)).to contain_exactly('failed', 'pending')
+        expect(builds_names).to eq ['build', 'test', 'deploy']
+        expect(builds_statuses).to eq ['success', 'success', 'pending']
+
+        succeed_running_or_pending
+
+        expect(builds_names).to eq ['build', 'test', 'deploy', 'cleanup']
+        expect(builds_statuses).to eq ['success', 'success', 'success', 'pending']
+
+        succeed_running_or_pending
+
+        expect(builds_statuses).to eq ['success', 'success', 'success', 'success']
+        expect(pipeline.reload.status).to eq 'success'
       end
     end
 
-    context 'properly creates builds when optional actions are defined' do
-      before do
-        create_build('build', stage_idx: 0)
-        create_build('test', stage_idx: 1)
-        create_build('test_failure', stage_idx: 2, when: 'on_failure')
-        create_build('deploy', stage_idx: 3)
-        create_build('production', stage_idx: 3, when: 'manual', allow_failure: true)
-        create_build('cleanup', stage_idx: 4, when: 'always')
-        create_build('clear cache', stage_idx: 4, when: 'manual', allow_failure: true)
-      end
+    context 'when test job fails' do
+      it 'properly creates builds' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names).to eq ['build']
+        expect(builds_statuses).to eq ['pending']
 
-      context 'when builds are successful' do
-        it 'properly creates builds' do
-          expect(process_pipeline).to be_truthy
-          expect(builds.pluck(:name)).to contain_exactly('build')
-          expect(builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy', 'cleanup')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'success', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:status)).to contain_exactly('success', 'success', 'success', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('success')
-        end
-      end
+        succeed_running_or_pending
 
-      context 'when test job fails' do
-        it 'properly creates builds' do
-          expect(process_pipeline).to be_truthy
-          expect(builds.pluck(:name)).to contain_exactly('build')
-          expect(builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'failed', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:status)).to contain_exactly('success', 'failed', 'success', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('failed')
-        end
-      end
+        expect(builds_names).to eq ['build', 'test']
+        expect(builds_statuses).to eq ['success', 'pending']
 
-      context 'when test and test_failure jobs fail' do
-        it 'properly creates builds' do
-          expect(process_pipeline).to be_truthy
-          expect(builds.pluck(:name)).to contain_exactly('build')
-          expect(builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'failed', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'failed', 'failed', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('failed')
-        end
-      end
+        fail_running_or_pending
 
-      context 'when deploy job fails' do
-        it 'properly creates builds' do
-          expect(process_pipeline).to be_truthy
-          expect(builds.pluck(:name)).to contain_exactly('build')
-          expect(builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy', 'cleanup')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'success', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(builds.pluck(:status)).to contain_exactly('success', 'success', 'failed', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('failed')
-        end
-      end
+        expect(builds_names).to eq ['build', 'test', 'test_failure']
+        expect(builds_statuses).to eq ['success', 'failed', 'pending']
 
-      context 'when build is canceled in the second stage' do
-        it 'does not schedule builds after build has been canceled' do
-          expect(process_pipeline).to be_truthy
-          expect(builds.pluck(:name)).to contain_exactly('build')
-          expect(builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
+        succeed_running_or_pending
 
-          expect(builds.running_or_pending).not_to be_empty
+        expect(builds_names).to eq ['build', 'test', 'test_failure', 'cleanup']
+        expect(builds_statuses).to eq ['success', 'failed', 'success', 'pending']
 
-          expect(builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:cancel)
+        succeed_running_or_pending
 
-          expect(builds.running_or_pending).to be_empty
-          expect(pipeline.reload.status).to eq('canceled')
-        end
+        expect(builds_statuses).to eq ['success', 'failed', 'success', 'success']
+        expect(pipeline.reload.status).to eq 'failed'
       end
+    end
+
+    context 'when test and test_failure jobs fail' do
+      it 'properly creates builds' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names).to eq ['build']
+        expect(builds_statuses).to eq ['pending']
+
+        succeed_running_or_pending
+
+        expect(builds_names).to eq ['build', 'test']
+        expect(builds_statuses).to eq ['success', 'pending']
+
+        fail_running_or_pending
 
-      context 'when listing optional manual actions' do
-        it 'returns only for skipped builds' do
-          # currently all builds are created
-          expect(process_pipeline).to be_truthy
-          expect(manual_actions).to be_empty
+        expect(builds_names).to eq ['build', 'test', 'test_failure']
+        expect(builds_statuses).to eq ['success', 'failed', 'pending']
 
-          # succeed stage build
-          pipeline.builds.running_or_pending.each(&:success)
-          expect(manual_actions).to be_empty
+        fail_running_or_pending
 
-          # succeed stage test
-          pipeline.builds.running_or_pending.each(&:success)
-          expect(manual_actions).to be_one # production
+        expect(builds_names).to eq ['build', 'test', 'test_failure', 'cleanup']
+        expect(builds_statuses).to eq ['success', 'failed', 'failed', 'pending']
 
-          # succeed stage deploy
-          pipeline.builds.running_or_pending.each(&:success)
-          expect(manual_actions).to be_many # production and clear cache
-        end
+        succeed_running_or_pending
+
+        expect(builds_names).to eq ['build', 'test', 'test_failure', 'cleanup']
+        expect(builds_statuses).to eq ['success', 'failed', 'failed', 'success']
+        expect(pipeline.reload.status).to eq('failed')
       end
     end
 
-    context 'when there are manual/on_failure jobs in earlier stages' do
-      before do
-        builds
-        process_pipeline
-        builds.each(&:reload)
-      end
+    context 'when deploy job fails' do
+      it 'properly creates builds' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names).to eq ['build']
+        expect(builds_statuses).to eq ['pending']
 
-      context 'when first stage has only optional manual actions' do
-        let(:builds) do
-          [create_build('build', stage_idx: 0, when: 'manual', allow_failure: true),
-           create_build('check', stage_idx: 1),
-           create_build('test', stage_idx: 2)]
-        end
+        succeed_running_or_pending
 
-        it 'starts from the second stage' do
-          expect(builds.map(&:status)).to eq(%w[skipped pending created])
-        end
-      end
+        expect(builds_names).to eq ['build', 'test']
+        expect(builds_statuses).to eq ['success', 'pending']
 
-      context 'when second stage has only optional manual actions' do
-        let(:builds) do
-          [create_build('check', stage_idx: 0),
-           create_build('build', stage_idx: 1, when: 'manual', allow_failure: true),
-           create_build('test', stage_idx: 2)]
-        end
+        succeed_running_or_pending
 
-        it 'skips second stage and continues on third stage' do
-          expect(builds.map(&:status)).to eq(%w[pending created created])
+        expect(builds_names).to eq ['build', 'test', 'deploy']
+        expect(builds_statuses).to eq ['success', 'success', 'pending']
 
-          builds.first.success
-          builds.each(&:reload)
+        fail_running_or_pending
 
-          expect(builds.map(&:status)).to eq(%w[success skipped pending])
-        end
+        expect(builds_names).to eq ['build', 'test', 'deploy', 'cleanup']
+        expect(builds_statuses).to eq ['success', 'success', 'failed', 'pending']
+
+        succeed_running_or_pending
+
+        expect(builds_statuses).to eq ['success', 'success', 'failed', 'success']
+        expect(pipeline.reload.status).to eq('failed')
       end
+    end
 
-      context 'when second stage has only on_failure jobs' do
-        let(:builds) do
-          [create_build('check', stage_idx: 0),
-           create_build('build', stage_idx: 1, when: 'on_failure'),
-           create_build('test', stage_idx: 2)]
-        end
+    context 'when build is canceled in the second stage' do
+      it 'does not schedule builds after build has been canceled' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names).to eq ['build']
+        expect(builds_statuses).to eq ['pending']
+
+        succeed_running_or_pending
 
-        it 'skips second stage and continues on third stage' do
-          expect(builds.map(&:status)).to eq(%w[pending created created])
+        expect(builds.running_or_pending).not_to be_empty
+        expect(builds_names).to eq ['build', 'test']
+        expect(builds_statuses).to eq ['success', 'pending']
 
-          builds.first.success
-          builds.each(&:reload)
+        cancel_running_or_pending
 
-          expect(builds.map(&:status)).to eq(%w[success skipped pending])
-        end
+        expect(builds.running_or_pending).to be_empty
+        expect(pipeline.reload.status).to eq 'canceled'
       end
     end
 
-    context 'when failed build in the middle stage is retried' do
-      context 'when failed build is the only unsuccessful build in the stage' do
-        before do
-          create_build('build:1', stage_idx: 0)
-          create_build('build:2', stage_idx: 0)
-          create_build('test:1', stage_idx: 1)
-          create_build('test:2', stage_idx: 1)
-          create_build('deploy:1', stage_idx: 2)
-          create_build('deploy:2', stage_idx: 2)
-        end
-
-        it 'does trigger builds in the next stage' do
-          expect(process_pipeline).to be_truthy
-          expect(builds.pluck(:name)).to contain_exactly('build:1', 'build:2')
+    context 'when listing optional manual actions' do
+      it 'returns only for skipped builds' do
+        # currently all builds are created
+        expect(process_pipeline).to be_truthy
+        expect(manual_actions).to be_empty
 
-          pipeline.builds.running_or_pending.each(&:success)
+        # succeed stage build
+        succeed_running_or_pending
 
-          expect(builds.pluck(:name))
-            .to contain_exactly('build:1', 'build:2', 'test:1', 'test:2')
+        expect(manual_actions).to be_empty
 
-          pipeline.builds.find_by(name: 'test:1').success
-          pipeline.builds.find_by(name: 'test:2').drop
+        # succeed stage test
+        succeed_running_or_pending
 
-          expect(builds.pluck(:name))
-            .to contain_exactly('build:1', 'build:2', 'test:1', 'test:2')
+        expect(manual_actions).to be_one # production
 
-          Ci::Build.retry(pipeline.builds.find_by(name: 'test:2'), user).success
+        # succeed stage deploy
+        succeed_running_or_pending
 
-          expect(builds.pluck(:name)).to contain_exactly(
-            'build:1', 'build:2', 'test:1', 'test:2', 'test:2', 'deploy:1', 'deploy:2')
-        end
+        expect(manual_actions).to be_many # production and clear cache
       end
     end
+  end
+
+  context 'when there are manual/on_failure jobs in earlier stages' do
+    context 'when first stage has only optional manual actions' do
+      before do
+        create_build('build', stage_idx: 0, when: 'manual', allow_failure: true)
+        create_build('check', stage_idx: 1)
+        create_build('test', stage_idx: 2)
 
-    context 'when there are builds that are not created yet' do
-      let(:pipeline) do
-        create(:ci_pipeline, config: config)
+        process_pipeline
       end
 
-      let(:config) do
-        { rspec: { stage: 'test', script: 'rspec' },
-          deploy: { stage: 'deploy', script: 'rsync' } }
+      it 'starts from the second stage' do
+        expect(all_builds_statuses).to eq %w[skipped pending created]
       end
+    end
 
+    context 'when second stage has only optional manual actions' do
       before do
-        create_build('linux', stage: 'build', stage_idx: 0)
-        create_build('mac', stage: 'build', stage_idx: 0)
+        create_build('check', stage_idx: 0)
+        create_build('build', stage_idx: 1, when: 'manual', allow_failure: true)
+        create_build('test', stage_idx: 2)
+
+        process_pipeline
       end
 
-      it 'processes the pipeline' do
-        # Currently we have five builds with state created
-        #
-        expect(builds.count).to eq(0)
-        expect(all_builds.count).to eq(2)
+      it 'skips second stage and continues on third stage' do
+        expect(all_builds_statuses).to eq(%w[pending created created])
 
-        # Process builds service will enqueue builds from the first stage.
-        #
-        process_pipeline
+        builds.first.success
+
+        expect(all_builds_statuses).to eq(%w[success skipped pending])
+      end
+    end
 
-        expect(builds.count).to eq(2)
-        expect(all_builds.count).to eq(2)
+    context 'when second stage has only on_failure jobs' do
+      before do
+        create_build('check', stage_idx: 0)
+        create_build('build', stage_idx: 1, when: 'on_failure')
+        create_build('test', stage_idx: 2)
 
-        # When builds succeed we will enqueue remaining builds.
-        #
-        # We will have 2 succeeded, 1 pending (from stage test), total 4 (two
-        # additional build from `.gitlab-ci.yml`).
-        #
-        succeed_pending
         process_pipeline
+      end
 
-        expect(builds.success.count).to eq(2)
-        expect(builds.pending.count).to eq(1)
-        expect(all_builds.count).to eq(4)
+      it 'skips second stage and continues on third stage' do
+        expect(all_builds_statuses).to eq(%w[pending created created])
 
-        # When pending merge_when_pipeline_succeeds in stage test, we enqueue deploy stage.
-        #
-        succeed_pending
-        process_pipeline
+        builds.first.success
 
-        expect(builds.pending.count).to eq(1)
-        expect(builds.success.count).to eq(3)
-        expect(all_builds.count).to eq(4)
+        expect(all_builds_statuses).to eq(%w[success skipped pending])
+      end
+    end
+  end
 
-        # When the last one succeeds we have 4 successful builds.
-        #
-        succeed_pending
-        process_pipeline
+  context 'when failed build in the middle stage is retried' do
+    context 'when failed build is the only unsuccessful build in the stage' do
+      before do
+        create_build('build:1', stage_idx: 0)
+        create_build('build:2', stage_idx: 0)
+        create_build('test:1', stage_idx: 1)
+        create_build('test:2', stage_idx: 1)
+        create_build('deploy:1', stage_idx: 2)
+        create_build('deploy:2', stage_idx: 2)
+      end
+
+      it 'does trigger builds in the next stage' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names).to eq ['build:1', 'build:2']
 
-        expect(builds.success.count).to eq(4)
-        expect(all_builds.count).to eq(4)
+        succeed_running_or_pending
+
+        expect(builds_names).to eq ['build:1', 'build:2', 'test:1', 'test:2']
+
+        pipeline.builds.find_by(name: 'test:1').success
+        pipeline.builds.find_by(name: 'test:2').drop
+
+        expect(builds_names).to eq ['build:1', 'build:2', 'test:1', 'test:2']
+
+        Ci::Build.retry(pipeline.builds.find_by(name: 'test:2'), user).success
+
+        expect(builds_names).to eq ['build:1', 'build:2', 'test:1', 'test:2',
+                                    'test:2', 'deploy:1', 'deploy:2']
       end
     end
   end
 
+  context 'when there are builds that are not created yet' do
+    let(:pipeline) do
+      create(:ci_pipeline, config: config)
+    end
+
+    let(:config) do
+      { rspec: { stage: 'test', script: 'rspec' },
+        deploy: { stage: 'deploy', script: 'rsync' } }
+    end
+
+    before do
+      create_build('linux', stage: 'build', stage_idx: 0)
+      create_build('mac', stage: 'build', stage_idx: 0)
+    end
+
+    it 'processes the pipeline' do
+      # Currently we have five builds with state created
+      #
+      expect(builds.count).to eq(0)
+      expect(all_builds.count).to eq(2)
+
+      # Process builds service will enqueue builds from the first stage.
+      #
+      process_pipeline
+
+      expect(builds.count).to eq(2)
+      expect(all_builds.count).to eq(2)
+
+      # When builds succeed we will enqueue remaining builds.
+      #
+      # We will have 2 succeeded, 1 pending (from stage test), total 4 (two
+      # additional build from `.gitlab-ci.yml`).
+      #
+      succeed_pending
+      process_pipeline
+
+      expect(builds.success.count).to eq(2)
+      expect(builds.pending.count).to eq(1)
+      expect(all_builds.count).to eq(4)
+
+      # When pending merge_when_pipeline_succeeds in stage test, we enqueue deploy stage.
+      #
+      succeed_pending
+      process_pipeline
+
+      expect(builds.pending.count).to eq(1)
+      expect(builds.success.count).to eq(3)
+      expect(all_builds.count).to eq(4)
+
+      # When the last one succeeds we have 4 successful builds.
+      #
+      succeed_pending
+      process_pipeline
+
+      expect(builds.success.count).to eq(4)
+      expect(all_builds.count).to eq(4)
+    end
+  end
+
   def all_builds
-    pipeline.builds
+    pipeline.builds.order(:stage_idx, :id)
   end
 
   def builds
     all_builds.where.not(status: [:created, :skipped])
   end
 
+  def builds_names
+    builds.pluck(:name)
+  end
+
+  def builds_statuses
+    builds.pluck(:status)
+  end
+
+  def all_builds_statuses
+    all_builds.pluck(:status)
+  end
+
   def process_pipeline
     described_class.new(pipeline.project, user).execute(pipeline)
   end
@@ -377,6 +403,18 @@ describe Ci::ProcessPipelineService, :services do
     builds.pending.update_all(status: 'success')
   end
 
+  def succeed_running_or_pending
+    pipeline.builds.running_or_pending.each(&:success)
+  end
+
+  def fail_running_or_pending
+    pipeline.builds.running_or_pending.each(&:drop)
+  end
+
+  def cancel_running_or_pending
+    pipeline.builds.running_or_pending.each(&:cancel)
+  end
+
   delegate :manual_actions, to: :pipeline
 
   def create_build(name, **opts)