Merge branch 'fix/gb/pipeline-retry-builds-started' into 'master'

Make pipeline retry take stages order into account

Closes #27431, #23710, and #26551

See merge request !9021

20 files changed, 529 insertions, 89 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 8c1b076c2d7..e018f8e7c4e 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -62,33 +62,10 @@ module Ci
         new_build.save
       end
 
-      def retry(build, user = nil)
-        new_build = Ci::Build.create(
-          ref: build.ref,
-          tag: build.tag,
-          options: build.options,
-          commands: build.commands,
-          tag_list: build.tag_list,
-          project: build.project,
-          pipeline: build.pipeline,
-          name: build.name,
-          allow_failure: build.allow_failure,
-          stage: build.stage,
-          stage_idx: build.stage_idx,
-          trigger_request: build.trigger_request,
-          yaml_variables: build.yaml_variables,
-          when: build.when,
-          user: user,
-          environment: build.environment,
-          status_event: 'enqueue'
-        )
-
-        MergeRequests::AddTodoWhenBuildFailsService
-          .new(build.project, nil)
-          .close(new_build)
-
-        build.pipeline.mark_as_processable_after_stage(build.stage_idx)
-        new_build
+      def retry(build, current_user)
+        Ci::RetryBuildService
+          .new(build.project, current_user)
+          .execute(build)
       end
     end
 
@@ -136,7 +113,7 @@ module Ci
       project.builds_enabled? && commands.present? && manual? && skipped?
     end
 
-    def play(current_user = nil)
+    def play(current_user)
       # Try to queue a current build
       if self.enqueue
         self.update(user: current_user)
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index bbc358adb83..dc4590a9923 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -214,21 +214,17 @@ module Ci
     def cancel_running
       Gitlab::OptimisticLocking.retry_lock(
         statuses.cancelable) do |cancelable|
-          cancelable.each(&:cancel)
+          cancelable.find_each(&:cancel)
         end
     end
 
-    def retry_failed(user)
-      Gitlab::OptimisticLocking.retry_lock(
-        builds.latest.failed_or_canceled) do |failed_or_canceled|
-          failed_or_canceled.select(&:retryable?).each do |build|
-            Ci::Build.retry(build, user)
-          end
-        end
+    def retry_failed(current_user)
+      Ci::RetryPipelineService.new(project, current_user)
+        .execute(self)
     end
 
     def mark_as_processable_after_stage(stage_idx)
-      builds.skipped.where('stage_idx > ?', stage_idx).find_each(&:process)
+      builds.skipped.after_stage(stage_idx).find_each(&:process)
     end
 
     def latest?
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 9547c57b2ae..99a6326309d 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -23,9 +23,6 @@ class CommitStatus < ActiveRecord::Base
     where(id: max_id.group(:name, :commit_id))
   end
 
-  scope :retried, -> { where.not(id: latest) }
-  scope :ordered, -> { order(:name) }
-
   scope :failed_but_allowed, -> do
     where(allow_failure: true, status: [:failed, :canceled])
   end
@@ -36,8 +33,11 @@ class CommitStatus < ActiveRecord::Base
       false, all_state_names - [:failed, :canceled])
   end
 
+  scope :retried, -> { where.not(id: latest) }
+  scope :ordered, -> { order(:name) }
   scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) }
   scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) }
+  scope :after_stage, -> (index) { where('stage_idx > ?', index) }
 
   state_machine :status do
     event :enqueue do
diff --git a/app/services/base_service.rb b/app/services/base_service.rb
index 1a2bad77a02..fa45506317e 100644
--- a/app/services/base_service.rb
+++ b/app/services/base_service.rb
@@ -1,4 +1,5 @@
 class BaseService
+  include Gitlab::Allowable
   include Gitlab::CurrentSettings
 
   attr_accessor :project, :current_user, :params
@@ -7,10 +8,6 @@ class BaseService
     @project, @current_user, @params = project, user, params.dup
   end
 
-  def can?(object, action, subject)
-    Ability.allowed?(object, action, subject)
-  end
-
   def notification_service
     NotificationService.new
   end
diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb
new file mode 100644
index 00000000000..4b47ee489cf
--- /dev/null
+++ b/app/services/ci/retry_build_service.rb
@@ -0,0 +1,42 @@
+module Ci
+  class RetryBuildService < ::BaseService
+    CLONE_ATTRIBUTES = %i[pipeline ref tag options commands tag_list name
+                          allow_failure stage stage_idx trigger_request
+                          yaml_variables when environment coverage_regex]
+                            .freeze
+
+    REJECT_ATTRIBUTES = %i[id status user token coverage trace runner
+                           artifacts_file artifacts_metadata artifacts_size
+                           created_at updated_at started_at finished_at
+                           queued_at erased_by erased_at].freeze
+
+    IGNORE_ATTRIBUTES = %i[trace type lock_version project target_url
+                           deploy job_id description].freeze
+
+    def execute(build)
+      reprocess(build).tap do |new_build|
+        build.pipeline.mark_as_processable_after_stage(build.stage_idx)
+
+        new_build.enqueue!
+
+        MergeRequests::AddTodoWhenBuildFailsService
+          .new(project, current_user)
+          .close(new_build)
+      end
+    end
+
+    def reprocess(build)
+      unless can?(current_user, :update_build, build)
+        raise Gitlab::Access::AccessDeniedError
+      end
+
+      attributes = CLONE_ATTRIBUTES.map do |attribute|
+        [attribute, build.send(attribute)]
+      end
+
+      attributes.push([:user, current_user])
+
+      project.builds.create(Hash[attributes])
+    end
+  end
+end
diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb
new file mode 100644
index 00000000000..2c5e130e5aa
--- /dev/null
+++ b/app/services/ci/retry_pipeline_service.rb
@@ -0,0 +1,22 @@
+module Ci
+  class RetryPipelineService < ::BaseService
+    def execute(pipeline)
+      unless can?(current_user, :update_pipeline, pipeline)
+        raise Gitlab::Access::AccessDeniedError
+      end
+
+      pipeline.builds.failed_or_canceled.find_each do |build|
+        next unless build.retryable?
+
+        Ci::RetryBuildService.new(project, current_user)
+          .reprocess(build)
+      end
+
+      MergeRequests::AddTodoWhenBuildFailsService
+        .new(project, current_user)
+        .close_all(pipeline)
+
+      pipeline.process!
+    end
+  end
+end
diff --git a/app/services/merge_requests/add_todo_when_build_fails_service.rb b/app/services/merge_requests/add_todo_when_build_fails_service.rb
index 12a8415d9a5..727768b1a39 100644
--- a/app/services/merge_requests/add_todo_when_build_fails_service.rb
+++ b/app/services/merge_requests/add_todo_when_build_fails_service.rb
@@ -18,5 +18,11 @@ module MergeRequests
         todo_service.merge_request_build_retried(merge_request)
       end
     end
+
+    def close_all(pipeline)
+      pipeline_merge_requests(pipeline) do |merge_request|
+        todo_service.merge_request_build_retried(merge_request)
+      end
+    end
   end
 end
diff --git a/changelogs/unreleased/fix-gb-pipeline-retry-builds-started.yml b/changelogs/unreleased/fix-gb-pipeline-retry-builds-started.yml
new file mode 100644
index 00000000000..49e243ca6bb
--- /dev/null
+++ b/changelogs/unreleased/fix-gb-pipeline-retry-builds-started.yml
@@ -0,0 +1,4 @@
+---
+title: Fix CI/CD pipeline retry and take stages order into account
+merge_request: 9021
+author:
diff --git a/features/steps/shared/builds.rb b/features/steps/shared/builds.rb
index d008a8a26af..5bc3a1f5ac4 100644
--- a/features/steps/shared/builds.rb
+++ b/features/steps/shared/builds.rb
@@ -11,7 +11,7 @@ module SharedBuilds
 
   step 'project has a recent build' do
     @pipeline = create(:ci_empty_pipeline, project: @project, sha: @project.commit.sha, ref: 'master')
-    @build = create(:ci_build_with_coverage, pipeline: @pipeline)
+    @build = create(:ci_build, :coverage, pipeline: @pipeline)
   end
 
   step 'recent build is successful' do
diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb
index e4cac0e1058..a90534d10ba 100644
--- a/spec/factories/ci/builds.rb
+++ b/spec/factories/ci/builds.rb
@@ -89,8 +89,9 @@ FactoryGirl.define do
       tag true
     end
 
-    factory :ci_build_with_coverage do
+    trait :coverage do
       coverage 99.9
+      coverage_regex '/(d+)/'
     end
 
     trait :trace do
@@ -99,6 +100,16 @@ FactoryGirl.define do
       end
     end
 
+    trait :erased do
+      erased_at Time.now
+      erased_by factory: :user
+    end
+
+    trait :queued do
+      queued_at Time.now
+      runner factory: :ci_runner
+    end
+
     trait :artifacts do
       after(:create) do |build, _|
         build.artifacts_file =
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 4080092405d..2dfca8bcfce 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -1,6 +1,7 @@
 require 'spec_helper'
 
 describe Ci::Build, :models do
+  let(:user) { create(:user) }
   let(:project) { create(:project, :repository) }
   let(:build) { create(:ci_build, pipeline: pipeline) }
   let(:test_trace) { 'This is a test' }
@@ -207,14 +208,16 @@ describe Ci::Build, :models do
     end
 
     it 'expects to have retried builds instead the original ones' do
-      retried_rspec = Ci::Build.retry(rspec_test)
-      expect(staging.depends_on_builds.map(&:id)).to contain_exactly(build.id, retried_rspec.id, rubocop_test.id)
+      project.add_developer(user)
+
+      retried_rspec = Ci::Build.retry(rspec_test, user)
+
+      expect(staging.depends_on_builds.map(&:id))
+        .to contain_exactly(build.id, retried_rspec.id, rubocop_test.id)
     end
   end
 
   describe '#detailed_status' do
-    let(:user) { create(:user) }
-
     it 'returns a detailed status' do
       expect(build.detailed_status(user))
         .to be_a Gitlab::Ci::Status::Build::Cancelable
@@ -813,12 +816,16 @@ describe Ci::Build, :models do
 
     subject { build.other_actions }
 
+    before do
+      project.add_developer(user)
+    end
+
     it 'returns other actions' do
       is_expected.to contain_exactly(other_build)
     end
 
     context 'when build is retried' do
-      let!(:new_build) { Ci::Build.retry(build) }
+      let!(:new_build) { Ci::Build.retry(build, user) }
 
       it 'does not return any of them' do
         is_expected.not_to include(build, new_build)
@@ -826,7 +833,7 @@ describe Ci::Build, :models do
     end
 
     context 'when other build is retried' do
-      let!(:retried_build) { Ci::Build.retry(other_build) }
+      let!(:retried_build) { Ci::Build.retry(other_build, user) }
 
       it 'returns a retried build' do
         is_expected.to contain_exactly(retried_build)
@@ -857,21 +864,29 @@ describe Ci::Build, :models do
   describe '#play' do
     let(:build) { create(:ci_build, :manual, pipeline: pipeline) }
 
-    subject { build.play }
+    before do
+      project.add_developer(user)
+    end
+
+    context 'when build is manual' do
+      it 'enqueues a build' do
+        new_build = build.play(user)
 
-    it 'enqueues a build' do
-      is_expected.to be_pending
-      is_expected.to eq(build)
+        expect(new_build).to be_pending
+        expect(new_build).to eq(build)
+      end
     end
 
-    context 'for successful build' do
+    context 'when build is passed' do
       before do
         build.update(status: 'success')
       end
 
       it 'creates a new build' do
-        is_expected.to be_pending
-        is_expected.not_to eq(build)
+        new_build = build.play(user)
+
+        expect(new_build).to be_pending
+        expect(new_build).not_to eq(build)
       end
     end
   end
@@ -1246,12 +1261,9 @@ describe Ci::Build, :models do
     end
 
     context 'when build has user' do
-      let(:user) { create(:user, username: 'starter') }
       let(:user_variables) do
-        [
-          { key: 'GITLAB_USER_ID',    value: user.id.to_s, public: true },
-          { key: 'GITLAB_USER_EMAIL', value: user.email,   public: true }
-        ]
+        [ { key: 'GITLAB_USER_ID',    value: user.id.to_s, public: true },
+          { key: 'GITLAB_USER_EMAIL', value: user.email,   public: true } ]
       end
 
       before do
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index 426be74cd02..10c2bfbb400 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -3,8 +3,12 @@ require 'spec_helper'
 describe Ci::Pipeline, models: true do
   include EmailHelpers
 
-  let(:project) { FactoryGirl.create :empty_project }
-  let(:pipeline) { FactoryGirl.create :ci_empty_pipeline, status: 'created', project: project }
+  let(:user) { create(:user) }
+  let(:project) { create(:empty_project) }
+
+  let(:pipeline) do
+    create(:ci_empty_pipeline, status: :created, project: project)
+  end
 
   it { is_expected.to belong_to(:project) }
   it { is_expected.to belong_to(:user) }
@@ -503,7 +507,9 @@ describe Ci::Pipeline, models: true do
   end
 
   describe '#status' do
-    let!(:build) { create(:ci_build, :created, pipeline: pipeline, name: 'test') }
+    let(:build) do
+      create(:ci_build, :created, pipeline: pipeline, name: 'test')
+    end
 
     subject { pipeline.reload.status }
 
@@ -545,13 +551,21 @@ describe Ci::Pipeline, models: true do
         build.cancel
       end
 
-      it { is_expected.to eq('canceled') }
+      context 'when build is pending' do
+        let(:build) do
+          create(:ci_build, :pending, pipeline: pipeline)
+        end
+
+        it { is_expected.to eq('canceled') }
+      end
     end
 
     context 'on failure and build retry' do
       before do
         build.drop
-        Ci::Build.retry(build)
+        project.add_developer(user)
+
+        Ci::Build.retry(build, user)
       end
 
       # We are changing a state: created > failed > running
@@ -563,8 +577,6 @@ describe Ci::Pipeline, models: true do
   end
 
   describe '#detailed_status' do
-    let(:user) { create(:user) }
-
     subject { pipeline.detailed_status(user) }
 
     context 'when pipeline is created' do
@@ -720,7 +732,7 @@ describe Ci::Pipeline, models: true do
   describe '#cancel_running' do
     let(:latest_status) { pipeline.statuses.pluck(:status) }
 
-    context 'when there is a running external job and created build' do
+    context 'when there is a running external job and a regular job' do
       before do
         create(:ci_build, :running, pipeline: pipeline)
         create(:generic_commit_status, :running, pipeline: pipeline)
@@ -733,7 +745,7 @@ describe Ci::Pipeline, models: true do
       end
     end
 
-    context 'when builds are in different stages' do
+    context 'when jobs are in different stages' do
       before do
         create(:ci_build, :running, stage_idx: 0, pipeline: pipeline)
         create(:ci_build, :running, stage_idx: 1, pipeline: pipeline)
@@ -745,17 +757,34 @@ describe Ci::Pipeline, models: true do
         expect(latest_status).to contain_exactly('canceled', 'canceled')
       end
     end
+
+    context 'when there are created builds present in the pipeline' do
+      before do
+        create(:ci_build, :running, stage_idx: 0, pipeline: pipeline)
+        create(:ci_build, :created, stage_idx: 1, pipeline: pipeline)
+
+        pipeline.cancel_running
+      end
+
+      it 'cancels created builds' do
+        expect(latest_status).to eq ['canceled', 'canceled']
+      end
+    end
   end
 
   describe '#retry_failed' do
     let(:latest_status) { pipeline.statuses.latest.pluck(:status) }
 
+    before do
+      project.add_developer(user)
+    end
+
     context 'when there is a failed build and failed external status' do
       before do
         create(:ci_build, :failed, name: 'build', pipeline: pipeline)
         create(:generic_commit_status, :failed, name: 'jenkins', pipeline: pipeline)
 
-        pipeline.retry_failed(create(:user))
+        pipeline.retry_failed(user)
       end
 
       it 'retries only build' do
@@ -768,11 +797,11 @@ describe Ci::Pipeline, models: true do
         create(:ci_build, :failed, name: 'build', stage_idx: 0, pipeline: pipeline)
         create(:ci_build, :failed, name: 'jenkins', stage_idx: 1, pipeline: pipeline)
 
-        pipeline.retry_failed(create(:user))
+        pipeline.retry_failed(user)
       end
 
       it 'retries both builds' do
-        expect(latest_status).to contain_exactly('pending', 'pending')
+        expect(latest_status).to contain_exactly('pending', 'created')
       end
     end
 
@@ -781,11 +810,11 @@ describe Ci::Pipeline, models: true do
         create(:ci_build, :failed, name: 'build', stage_idx: 0, pipeline: pipeline)
         create(:ci_build, :canceled, name: 'jenkins', stage_idx: 1, pipeline: pipeline)
 
-        pipeline.retry_failed(create(:user))
+        pipeline.retry_failed(user)
       end
 
       it 'retries both builds' do
-        expect(latest_status).to contain_exactly('pending', 'pending')
+        expect(latest_status).to contain_exactly('pending', 'created')
       end
     end
   end
diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb
index bf4394f7d5b..36533bdd11e 100644
--- a/spec/models/commit_status_spec.rb
+++ b/spec/models/commit_status_spec.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe CommitStatus, models: true do
+describe CommitStatus, :models do
   let(:project) { create(:project, :repository) }
 
   let(:pipeline) do
@@ -127,7 +127,7 @@ describe CommitStatus, models: true do
   end
 
   describe '.latest' do
-    subject { CommitStatus.latest.order(:id) }
+    subject { described_class.latest.order(:id) }
 
     let(:statuses) do
       [create_status(name: 'aa', ref: 'bb', status: 'running'),
@@ -143,7 +143,7 @@ describe CommitStatus, models: true do
   end
 
   describe '.running_or_pending' do
-    subject { CommitStatus.running_or_pending.order(:id) }
+    subject { described_class.running_or_pending.order(:id) }
 
     let(:statuses) do
       [create_status(name: 'aa', ref: 'bb', status: 'running'),
@@ -159,7 +159,21 @@ describe CommitStatus, models: true do
   end
 
   describe '.exclude_ignored' do
-    subject { CommitStatus.exclude_ignored.order(:id) }
+    subject { described_class.after_stage(0) }
+
+    let(:statuses) do
+      [create_status(name: 'aa', stage_idx: 0),
+       create_status(name: 'cc', stage_idx: 1),
+       create_status(name: 'aa', stage_idx: 2)]
+    end
+
+    it 'returns statuses from second and third stage' do
+      is_expected.to eq(statuses.values_at(1, 2))
+    end
+  end
+
+  describe '.exclude_ignored' do
+    subject { described_class.exclude_ignored.order(:id) }
 
     let(:statuses) do
       [create_status(when: 'manual', status: 'skipped'),
diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb
index 960f29f3805..f0ed0c679d5 100644
--- a/spec/models/environment_spec.rb
+++ b/spec/models/environment_spec.rb
@@ -155,7 +155,7 @@ describe Environment, models: true do
   end
 
   describe '#stop_with_action!' do
-    let(:user) { create(:user) }
+    let(:user) { create(:admin) }
 
     subject { environment.stop_with_action!(user) }
 
diff --git a/spec/requests/api/commit_statuses_spec.rb b/spec/requests/api/commit_statuses_spec.rb
index 1e4cb611681..5cba546bee2 100644
--- a/spec/requests/api/commit_statuses_spec.rb
+++ b/spec/requests/api/commit_statuses_spec.rb
@@ -5,12 +5,15 @@ describe API::CommitStatuses, api: true do
 
   let!(:project) { create(:project, :repository) }
   let(:commit) { project.repository.commit }
-  let(:commit_status) { create(:commit_status, pipeline: pipeline) }
   let(:guest) { create_user(:guest) }
   let(:reporter) { create_user(:reporter) }
   let(:developer) { create_user(:developer) }
   let(:sha) { commit.id }
 
+  let(:commit_status) do
+    create(:commit_status, status: :pending, pipeline: pipeline)
+  end
+
   describe "GET /projects/:id/repository/commits/:sha/statuses" do
     let(:get_url) { "/projects/#{project.id}/repository/commits/#{sha}/statuses" }
 
diff --git a/spec/services/ci/process_pipeline_service_spec.rb b/spec/services/ci/process_pipeline_service_spec.rb
index ebb11166964..ef2ddc4b1d7 100644
--- a/spec/services/ci/process_pipeline_service_spec.rb
+++ b/spec/services/ci/process_pipeline_service_spec.rb
@@ -1,8 +1,16 @@
 require 'spec_helper'
 
-describe Ci::ProcessPipelineService, services: true do
-  let(:pipeline) { create(:ci_empty_pipeline, ref: 'master') }
+describe Ci::ProcessPipelineService, :services do
   let(:user) { create(:user) }
+  let(:project) { create(:empty_project) }
+
+  let(:pipeline) do
+    create(:ci_empty_pipeline, ref: 'master', project: project)
+  end
+
+  before do
+    project.add_developer(user)
+  end
 
   describe '#execute' do
     context 'start queuing next builds' do
@@ -285,7 +293,7 @@ describe Ci::ProcessPipelineService, services: true do
           expect(builds.pluck(:name))
             .to contain_exactly('build:1', 'build:2', 'test:1', 'test:2')
 
-          Ci::Build.retry(pipeline.builds.find_by(name: 'test:2')).success
+          Ci::Build.retry(pipeline.builds.find_by(name: 'test:2'), user).success
 
           expect(builds.pluck(:name)).to contain_exactly(
             'build:1', 'build:2', 'test:1', 'test:2', 'test:2', 'deploy:1', 'deploy:2')
diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb
new file mode 100644
index 00000000000..93147870afe
--- /dev/null
+++ b/spec/services/ci/retry_build_service_spec.rb
@@ -0,0 +1,117 @@
+require 'spec_helper'
+
+describe Ci::RetryBuildService, :services do
+  let(:user) { create(:user) }
+  let(:project) { create(:empty_project) }
+  let(:pipeline) { create(:ci_pipeline, project: project) }
+  let(:build) { create(:ci_build, pipeline: pipeline) }
+
+  let(:service) do
+    described_class.new(project, user)
+  end
+
+  shared_examples 'build duplication' do
+    let(:build) do
+      create(:ci_build, :failed, :artifacts, :erased, :trace,
+             :queued, :coverage, pipeline: pipeline)
+    end
+
+    describe 'clone attributes' do
+      described_class::CLONE_ATTRIBUTES.each do |attribute|
+        it "clones #{attribute} build attribute" do
+          expect(new_build.send(attribute)).to eq build.send(attribute)
+        end
+      end
+    end
+
+    describe 'reject attributes' do
+      described_class::REJECT_ATTRIBUTES.each do |attribute|
+        it "does not clone #{attribute} build attribute" do
+          expect(new_build.send(attribute)).not_to eq build.send(attribute)
+        end
+      end
+    end
+
+    it 'has correct number of known attributes' do
+      attributes =
+        described_class::CLONE_ATTRIBUTES +
+        described_class::IGNORE_ATTRIBUTES +
+        described_class::REJECT_ATTRIBUTES
+
+      expect(attributes.size).to eq build.attributes.size
+    end
+  end
+
+  describe '#execute' do
+    let(:new_build) { service.execute(build) }
+
+    context 'when user has ability to execute build' do
+      before do
+        project.add_developer(user)
+      end
+
+      it_behaves_like 'build duplication'
+
+      it 'creates a new build that represents the old one' do
+        expect(new_build.name).to eq build.name
+      end
+
+      it 'enqueues the new build' do
+        expect(new_build).to be_pending
+      end
+
+      it 'resolves todos for old build that failed' do
+        expect(MergeRequests::AddTodoWhenBuildFailsService)
+          .to receive_message_chain(:new, :close)
+
+        service.execute(build)
+      end
+
+      context 'when there are subsequent builds that are skipped' do
+        let!(:subsequent_build) do
+          create(:ci_build, :skipped, stage_idx: 1, pipeline: pipeline)
+        end
+
+        it 'resumes pipeline processing in subsequent stages' do
+          service.execute(build)
+
+          expect(subsequent_build.reload).to be_created
+        end
+      end
+    end
+
+    context 'when user does not have ability to execute build' do
+      it 'raises an error' do
+        expect { service.execute(build) }
+          .to raise_error Gitlab::Access::AccessDeniedError
+      end
+    end
+  end
+
+  describe '#reprocess' do
+    let(:new_build) { service.reprocess(build) }
+
+    context 'when user has ability to execute build' do
+      before do
+        project.add_developer(user)
+      end
+
+      it_behaves_like 'build duplication'
+
+      it 'creates a new build that represents the old one' do
+        expect(new_build.name).to eq build.name
+      end
+
+      it 'does not enqueue the new build' do
+        expect(new_build).to be_created
+      end
+    end
+
+    context 'when user does not have ability to execute build' do
+      it 'raises an error' do
+        expect { service.reprocess(build) }
+          .to raise_error Gitlab::Access::AccessDeniedError
+      end
+    end
+  end
+end
diff --git a/spec/services/ci/retry_pipeline_service_spec.rb b/spec/services/ci/retry_pipeline_service_spec.rb
new file mode 100644
index 00000000000..c0af8b8450a
--- /dev/null
+++ b/spec/services/ci/retry_pipeline_service_spec.rb
@@ -0,0 +1,175 @@
+require 'spec_helper'
+
+describe Ci::RetryPipelineService, '#execute', :services do
+  let(:user) { create(:user) }
+  let(:project) { create(:empty_project) }
+  let(:pipeline) { create(:ci_pipeline, project: project) }
+  let(:service) { described_class.new(project, user) }
+
+  context 'when user has ability to modify pipeline' do
+    let(:user) { create(:admin) }
+
+    context 'when there are failed builds in the last stage' do
+      before do
+        create_build('rspec 1', :success, 0)
+        create_build('rspec 2', :failed, 1)
+        create_build('rspec 3', :canceled, 1)
+      end
+
+      it 'enqueues all builds in the last stage' do
+        service.execute(pipeline)
+
+        expect(build('rspec 2')).to be_pending
+        expect(build('rspec 3')).to be_pending
+        expect(pipeline.reload).to be_running
+      end
+    end
+
+    context 'when there are failed or canceled builds in the first stage' do
+      before do
+        create_build('rspec 1', :failed, 0)
+        create_build('rspec 2', :canceled, 0)
+        create_build('rspec 3', :canceled, 1)
+        create_build('spinach 1', :canceled, 2)
+      end
+
+      it 'retries builds failed builds and marks subsequent for processing' do
+        service.execute(pipeline)
+
+        expect(build('rspec 1')).to be_pending
+        expect(build('rspec 2')).to be_pending
+        expect(build('rspec 3')).to be_created
+        expect(build('spinach 1')).to be_created
+        expect(pipeline.reload).to be_running
+      end
+    end
+
+    context 'when there is failed build present which was run on failure' do
+      before do
+        create_build('rspec 1', :failed, 0)
+        create_build('rspec 2', :canceled, 0)
+        create_build('rspec 3', :canceled, 1)
+        create_build('report 1', :failed, 2)
+      end
+
+      it 'retries builds only in the first stage' do
+        service.execute(pipeline)
+
+        expect(build('rspec 1')).to be_pending
+        expect(build('rspec 2')).to be_pending
+        expect(build('rspec 3')).to be_created
+        expect(build('report 1')).to be_created
+        expect(pipeline.reload).to be_running
+      end
+
+      it 'creates a new job for report job in this case' do
+        service.execute(pipeline)
+
+        expect(statuses.where(name: 'report 1').first).to be_retried
+      end
+    end
+
+    context 'when pipeline contains manual actions' do
+      context 'when there is a canceled manual action in first stage' do
+        before do
+          create_build('rspec 1', :failed, 0)
+          create_build('staging', :canceled, 0, :manual)
+          create_build('rspec 2', :canceled, 1)
+        end
+
+        it 'retries builds failed builds and marks subsequent for processing' do
+          service.execute(pipeline)
+
+          expect(build('rspec 1')).to be_pending
+          expect(build('staging')).to be_skipped
+          expect(build('rspec 2')).to be_created
+          expect(pipeline.reload).to be_running
+        end
+      end
+
+      context 'when there is a skipped manual action in last stage' do
+        before do
+          create_build('rspec 1', :canceled, 0)
+          create_build('staging', :skipped, 1, :manual)
+        end
+
+        it 'retries canceled job and skips manual action' do
+          service.execute(pipeline)
+
+          expect(build('rspec 1')).to be_pending
+          expect(build('staging')).to be_skipped
+          expect(pipeline.reload).to be_running
+        end
+      end
+
+      context 'when there is a created manual action in the last stage' do
+        before do
+          create_build('rspec 1', :canceled, 0)
+          create_build('staging', :created, 1, :manual)
+        end
+
+        it 'retries canceled job and does not update the manual action' do
+          service.execute(pipeline)
+
+          expect(build('rspec 1')).to be_pending
+          expect(build('staging')).to be_created
+          expect(pipeline.reload).to be_running
+        end
+      end
+
+      context 'when there is a created manual action in the first stage' do
+        before do
+          create_build('rspec 1', :canceled, 0)
+          create_build('staging', :created, 0, :manual)
+        end
+
+        it 'retries canceled job and skipps the manual action' do
+          service.execute(pipeline)
+
+          expect(build('rspec 1')).to be_pending
+          expect(build('staging')).to be_skipped
+          expect(pipeline.reload).to be_running
+        end
+      end
+    end
+
+    it 'closes all todos about failed jobs for pipeline' do
+      expect(MergeRequests::AddTodoWhenBuildFailsService)
+        .to receive_message_chain(:new, :close_all)
+
+      service.execute(pipeline)
+    end
+
+    it 'reprocesses the pipeline' do
+      expect(pipeline).to receive(:process!)
+
+      service.execute(pipeline)
+    end
+  end
+
+  context 'when user is not allowed to retry pipeline' do
+    it 'raises an error' do
+      expect { service.execute(pipeline) }
+        .to raise_error Gitlab::Access::AccessDeniedError
+    end
+  end
+
+  def statuses
+    pipeline.reload.statuses
+  end
+
+  def build(name)
+    statuses.latest.find_by(name: name)
+  end
+
+  def create_build(name, status, stage_num, on = 'on_success')
+    create(:ci_build, name: name,
+                      status: status,
+                      stage: "stage_#{stage_num}",
+                      stage_idx: stage_num,
+                      when: on,
+                      pipeline: pipeline) do |build|
+      pipeline.update_status
+    end
+  end
+end
diff --git a/spec/services/create_deployment_service_spec.rb b/spec/services/create_deployment_service_spec.rb
index cf0a18aacec..6fb4d517115 100644
--- a/spec/services/create_deployment_service_spec.rb
+++ b/spec/services/create_deployment_service_spec.rb
@@ -234,7 +234,11 @@ describe CreateDeploymentService, services: true do
 
       context 'when build is retried' do
         it_behaves_like 'does create environment and deployment' do
-          let(:deployable) { Ci::Build.retry(build) }
+          before do
+            project.add_developer(user)
+          end
+
+          let(:deployable) { Ci::Build.retry(build, user) }
 
           subject { deployable.success }
         end
diff --git a/spec/services/merge_requests/add_todo_when_build_fails_service_spec.rb b/spec/services/merge_requests/add_todo_when_build_fails_service_spec.rb
index bb7830c7eea..d80fb8a1af1 100644
--- a/spec/services/merge_requests/add_todo_when_build_fails_service_spec.rb
+++ b/spec/services/merge_requests/add_todo_when_build_fails_service_spec.rb
@@ -17,7 +17,7 @@ describe MergeRequests::AddTodoWhenBuildFailsService do
     described_class.new(project, user, commit_message: 'Awesome message')
   end
 
-  let(:todo_service) { TodoService.new }
+  let(:todo_service) { spy('todo service') }
 
   let(:merge_request) do
     create(:merge_request, merge_user: user,
@@ -107,4 +107,27 @@ describe MergeRequests::AddTodoWhenBuildFailsService do
       end
     end
   end
+
+  describe '#close_all' do
+    context 'when using pipeline that belongs to merge request' do
+      it 'resolves todos about failed builds for pipeline' do
+        service.close_all(pipeline)
+
+        expect(todo_service)
+          .to have_received(:merge_request_build_retried)
+          .with(merge_request)
+      end
+    end
+
+    context 'when pipeline is not related to merge request' do
+      let(:pipeline) { create(:ci_empty_pipeline) }
+
+      it 'does not resolve any todos about failed builds' do
+        service.close_all(pipeline)
+
+        expect(todo_service)
+          .not_to have_received(:merge_request_build_retried)
+      end
+    end
+  end
 end