Merge branch 'scheduled-manual-jobs' into 'master'

Delayed jobs

Closes #51352

See merge request gitlab-org/gitlab-ce!21767

33 files changed, 310 insertions, 81 deletions

diff --git a/app/assets/images/ci_favicons/canary/favicon_status_scheduled.ico b/app/assets/images/ci_favicons/canary/favicon_status_scheduled.ico
new file mode 100644
index 00000000000..5444b8e41dc
--- /dev/null
+++ b/app/assets/images/ci_favicons/canary/favicon_status_scheduled.ico
diff --git a/app/assets/images/ci_favicons/favicon_status_scheduled.png b/app/assets/images/ci_favicons/favicon_status_scheduled.png
new file mode 100644
index 00000000000..d198c255fdd
--- /dev/null
+++ b/app/assets/images/ci_favicons/favicon_status_scheduled.png
diff --git a/app/assets/javascripts/lib/utils/datetime_utility.js b/app/assets/javascripts/lib/utils/datetime_utility.js
index 1f66fa811ea..833dbefd3dc 100644
--- a/app/assets/javascripts/lib/utils/datetime_utility.js
+++ b/app/assets/javascripts/lib/utils/datetime_utility.js
@@ -370,3 +370,24 @@ window.gl.utils = {
   getTimeago,
   localTimeAgo,
 };
+
+/**
+ * Formats milliseconds as timestamp (e.g. 01:02:03).
+ * This takes durations longer than a day into account (e.g. two days would be 48:00:00).
+ *
+ * @param milliseconds
+ * @returns {string}
+ */
+export const formatTime = milliseconds => {
+  const remainingSeconds = Math.floor(milliseconds / 1000) % 60;
+  const remainingMinutes = Math.floor(milliseconds / 1000 / 60) % 60;
+  const remainingHours = Math.floor(milliseconds / 1000 / 60 / 60);
+  let formattedTime = '';
+  if (remainingHours < 10) formattedTime += '0';
+  formattedTime += `${remainingHours}:`;
+  if (remainingMinutes < 10) formattedTime += '0';
+  formattedTime += `${remainingMinutes}:`;
+  if (remainingSeconds < 10) formattedTime += '0';
+  formattedTime += remainingSeconds;
+  return formattedTime;
+};
diff --git a/app/assets/javascripts/pipelines/components/pipelines_actions.vue b/app/assets/javascripts/pipelines/components/pipelines_actions.vue
index 017dd560621..16e69759091 100644
--- a/app/assets/javascripts/pipelines/components/pipelines_actions.vue
+++ b/app/assets/javascripts/pipelines/components/pipelines_actions.vue
@@ -1,4 +1,6 @@
 <script>
+import { s__, sprintf } from '~/locale';
+import { formatTime } from '~/lib/utils/datetime_utility';
 import eventHub from '../event_hub';
 import icon from '../../vue_shared/components/icon.vue';
 import tooltip from '../../vue_shared/directives/tooltip';
@@ -22,10 +24,24 @@ export default {
     };
   },
   methods: {
-    onClickAction(endpoint) {
+    onClickAction(action) {
+      if (action.scheduled_at) {
+        const confirmationMessage = sprintf(
+          s__(
+            "DelayedJobs|Are you sure you want to run %{jobName} immediately? This job will run automatically after it's timer finishes.",
+          ),
+          { jobName: action.name },
+        );
+        // https://gitlab.com/gitlab-org/gitlab-ce/issues/52156
+        // eslint-disable-next-line no-alert
+        if (!window.confirm(confirmationMessage)) {
+          return;
+        }
+      }
+
       this.isLoading = true;
 
-      eventHub.$emit('postAction', endpoint);
+      eventHub.$emit('postAction', action.path);
     },
 
     isActionDisabled(action) {
@@ -35,6 +51,11 @@ export default {
 
       return !action.playable;
     },
+
+    remainingTime(action) {
+      const remainingMilliseconds = new Date(action.scheduled_at).getTime() - Date.now();
+      return formatTime(Math.max(0, remainingMilliseconds));
+    },
   },
 };
 </script>
@@ -63,17 +84,24 @@ export default {
 
     <ul class="dropdown-menu dropdown-menu-right">
       <li
-        v-for="(action, i) in actions"
-        :key="i"
+        v-for="action in actions"
+        :key="action.path"
       >
         <button
           :class="{ disabled: isActionDisabled(action) }"
           :disabled="isActionDisabled(action)"
           type="button"
           class="js-pipeline-action-link no-btn btn"
-          @click="onClickAction(action.path)"
+          @click="onClickAction(action)"
         >
           {{ action.name }}
+          <span
+            v-if="action.scheduled_at"
+            class="pull-right"
+          >
+            <icon name="clock" />
+            {{ remainingTime(action) }}
+          </span>
         </button>
       </li>
     </ul>
diff --git a/app/assets/javascripts/pipelines/components/pipelines_table_row.vue b/app/assets/javascripts/pipelines/components/pipelines_table_row.vue
index a39cc265601..09ee190b8ca 100644
--- a/app/assets/javascripts/pipelines/components/pipelines_table_row.vue
+++ b/app/assets/javascripts/pipelines/components/pipelines_table_row.vue
@@ -59,6 +59,16 @@ export default {
     };
   },
   computed: {
+    actions() {
+      if (!this.pipeline || !this.pipeline.details) {
+        return [];
+      }
+      const { details } = this.pipeline;
+      return [
+        ...(details.manual_actions || []),
+        ...(details.scheduled_actions || []),
+      ];
+    },
     /**
      * If provided, returns the commit tag.
      * Needed to render the commit component column.
@@ -321,8 +331,8 @@ export default {
     >
       <div class="btn-group table-action-buttons">
         <pipelines-actions-component
-          v-if="pipeline.details.manual_actions.length"
-          :actions="pipeline.details.manual_actions"
+          v-if="actions.length > 0"
+          :actions="actions"
         />
 
         <pipelines-artifacts-component
diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss
index 686ce0c63a4..c4296c7a88a 100644
--- a/app/assets/stylesheets/framework/buttons.scss
+++ b/app/assets/stylesheets/framework/buttons.scss
@@ -360,6 +360,10 @@
   i {
     color: $gl-text-color-secondary;
   }
+
+  svg {
+    fill: $gl-text-color-secondary;
+  }
 }
 
 .clone-dropdown-btn a {
diff --git a/app/assets/stylesheets/framework/icons.scss b/app/assets/stylesheets/framework/icons.scss
index f002edced8a..abd26e38d18 100644
--- a/app/assets/stylesheets/framework/icons.scss
+++ b/app/assets/stylesheets/framework/icons.scss
@@ -64,6 +64,7 @@
   }
 }
 
+.ci-status-icon-scheduled,
 .ci-status-icon-manual {
   svg {
     fill: $gl-text-color;
diff --git a/app/assets/stylesheets/pages/pipelines.scss b/app/assets/stylesheets/pages/pipelines.scss
index 8bb8b83dc5e..14395cc59b0 100644
--- a/app/assets/stylesheets/pages/pipelines.scss
+++ b/app/assets/stylesheets/pages/pipelines.scss
@@ -760,6 +760,7 @@
   }
 
   &.ci-status-icon-canceled,
+  &.ci-status-icon-scheduled,
   &.ci-status-icon-disabled,
   &.ci-status-icon-not-found,
   &.ci-status-icon-manual {
diff --git a/app/assets/stylesheets/pages/status.scss b/app/assets/stylesheets/pages/status.scss
index 620297e589d..7d59dd3b5d1 100644
--- a/app/assets/stylesheets/pages/status.scss
+++ b/app/assets/stylesheets/pages/status.scss
@@ -27,6 +27,7 @@
 
   &.ci-canceled,
   &.ci-disabled,
+  &.ci-scheduled,
   &.ci-manual {
     color: $gl-text-color;
     border-color: $gl-text-color;
diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb
index 3f85e442be9..9c9bbe04947 100644
--- a/app/controllers/projects/jobs_controller.rb
+++ b/app/controllers/projects/jobs_controller.rb
@@ -110,6 +110,13 @@ class Projects::JobsController < Projects::ApplicationController
     redirect_to build_path(@build)
   end
 
+  def unschedule
+    return respond_422 unless @build.scheduled?
+
+    @build.unschedule!
+    redirect_to build_path(@build)
+  end
+
   def status
     render json: BuildSerializer
       .new(project: @project, current_user: @current_user)
diff --git a/app/helpers/ci_status_helper.rb b/app/helpers/ci_status_helper.rb
index 136772e1ec3..6f9e2ef78cd 100644
--- a/app/helpers/ci_status_helper.rb
+++ b/app/helpers/ci_status_helper.rb
@@ -20,6 +20,8 @@ module CiStatusHelper
               'passed with warnings'
             when 'manual'
               'waiting for manual action'
+            when 'scheduled'
+              'waiting for delayed job'
             else
               status
             end
@@ -39,6 +41,8 @@ module CiStatusHelper
       s_('CiStatusText|passed')
     when 'manual'
       s_('CiStatusText|blocked')
+    when 'scheduled'
+      s_('CiStatusText|scheduled')
     else
       # All states are already being translated inside the detailed statuses:
       # :running => Gitlab::Ci::Status::Running
@@ -83,6 +87,8 @@ module CiStatusHelper
         'status_skipped'
       when 'manual'
         'status_manual'
+      when 'scheduled'
+        'status_scheduled'
       else
         'status_canceled'
       end
diff --git a/app/helpers/time_helper.rb b/app/helpers/time_helper.rb
index 94044d7b85e..3e6a301b77d 100644
--- a/app/helpers/time_helper.rb
+++ b/app/helpers/time_helper.rb
@@ -21,9 +21,17 @@ module TimeHelper
     "#{from.to_s(:short)} - #{to.to_s(:short)}"
   end
 
-  def duration_in_numbers(duration)
-    time_format = duration < 1.hour ? "%M:%S" : "%H:%M:%S"
+  def duration_in_numbers(duration_in_seconds, allow_overflow = false)
+    if allow_overflow
+      seconds = duration_in_seconds % 1.minute
+      minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute)
+      hours = duration_in_seconds / 1.hour
 
-    Time.at(duration).utc.strftime(time_format)
+      "%02d:%02d:%02d" % [hours, minutes, seconds]
+    else
+      time_format = duration_in_seconds < 1.hour ? "%M:%S" : "%H:%M:%S"
+
+      Time.at(duration_in_seconds).utc.strftime(time_format)
+    end
   end
 end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index a59ff731954..cdfe8175a42 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -92,7 +92,8 @@ module Ci
     scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }
     scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
-    scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
+    scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) }
+    scope :scheduled_actions, ->() { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) }
     scope :ref_protected, -> { where(protected: true) }
     scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) }
 
@@ -159,6 +160,34 @@ module Ci
         transition created: :manual
       end
 
+      event :schedule do
+        transition created: :scheduled
+      end
+
+      event :unschedule do
+        transition scheduled: :manual
+      end
+
+      event :enqueue_scheduled do
+        transition scheduled: :pending, if: ->(build) do
+          build.scheduled_at && build.scheduled_at < Time.now
+        end
+      end
+
+      before_transition scheduled: any do |build|
+        build.scheduled_at = nil
+      end
+
+      before_transition created: :scheduled do |build|
+        build.scheduled_at = build.options_scheduled_at
+      end
+
+      after_transition created: :scheduled do |build|
+        build.run_after_commit do
+          Ci::BuildScheduleWorker.perform_at(build.scheduled_at, build.id)
+        end
+      end
+
       after_transition any => [:pending] do |build|
         build.run_after_commit do
           BuildQueueWorker.perform_async(id)
@@ -226,11 +255,20 @@ module Ci
     end
 
     def playable?
-      action? && (manual? || retryable?)
+      action? && (manual? || scheduled? || retryable?)
+    end
+
+    def schedulable?
+      Feature.enabled?('ci_enable_scheduled_build', default_enabled: true) &&
+        self.when == 'delayed' && options[:start_in].present?
+    end
+
+    def options_scheduled_at
+      ChronicDuration.parse(options[:start_in])&.seconds&.from_now
     end
 
     def action?
-      self.when == 'manual'
+      %w[manual delayed].include?(self.when)
     end
 
     # rubocop: disable CodeReuse/ServiceClass
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 69def660e8e..17024e8a0af 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -35,6 +35,7 @@ module Ci
     has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
     has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
     has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
     has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
 
     has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
@@ -80,7 +81,7 @@ module Ci
 
     state_machine :status, initial: :created do
       event :enqueue do
-        transition [:created, :skipped] => :pending
+        transition [:created, :skipped, :scheduled] => :pending
         transition [:success, :failed, :canceled] => :running
       end
 
@@ -108,6 +109,10 @@ module Ci
         transition any - [:manual] => :manual
       end
 
+      event :delay do
+        transition any - [:scheduled] => :scheduled
+      end
+
       # IMPORTANT
       # Do not add any operations to this state_machine
       # Create a separate worker for each new operation
@@ -544,6 +549,7 @@ module Ci
         when 'canceled' then cancel
         when 'skipped' then skip
         when 'manual' then block
+        when 'scheduled' then delay
         else
           raise HasStatus::UnknownStatusError,
                 "Unknown status `#{latest_builds_status}`"
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 511ded55dc3..58f3fe2460a 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -65,6 +65,10 @@ module Ci
       event :block do
         transition any - [:manual] => :manual
       end
+
+      event :delay do
+        transition any - [:scheduled] => :scheduled
+      end
     end
 
     def update_status
@@ -77,6 +81,7 @@ module Ci
         when 'failed' then drop
         when 'canceled' then cancel
         when 'manual' then block
+        when 'scheduled' then delay
         when 'skipped', nil then skip
         else
           raise HasStatus::UnknownStatusError,
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index fe2f144ef03..06507345fe8 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -49,7 +49,8 @@ class CommitStatus < ActiveRecord::Base
     stuck_or_timeout_failure: 3,
     runner_system_failure: 4,
     missing_dependency_failure: 5,
-    runner_unsupported: 6
+    runner_unsupported: 6,
+    stale_schedule: 7
   }
 
   ##
@@ -71,7 +72,7 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :enqueue do
-      transition [:created, :skipped, :manual] => :pending
+      transition [:created, :skipped, :manual, :scheduled] => :pending
     end
 
     event :run do
@@ -83,7 +84,7 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :drop do
-      transition [:created, :pending, :running] => :failed
+      transition [:created, :pending, :running, :scheduled] => :failed
     end
 
     event :success do
@@ -91,10 +92,10 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :cancel do
-      transition [:created, :pending, :running, :manual] => :canceled
+      transition [:created, :pending, :running, :manual, :scheduled] => :canceled
     end
 
-    before_transition [:created, :skipped, :manual] => :pending do |commit_status|
+    before_transition [:created, :skipped, :manual, :scheduled] => :pending do |commit_status|
       commit_status.queued_at = Time.now
     end
 
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index b3960cbad1a..b92643f87f8 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -4,14 +4,15 @@ module HasStatus
   extend ActiveSupport::Concern
 
   DEFAULT_STATUS = 'created'.freeze
-  BLOCKED_STATUS = 'manual'.freeze
-  AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual].freeze
-  STARTED_STATUSES = %w[running success failed skipped manual].freeze
+  BLOCKED_STATUS = %w[manual scheduled].freeze
+  AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual scheduled].freeze
+  STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze
   ACTIVE_STATUSES = %w[pending running].freeze
   COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
-  ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze
+  ORDERED_STATUSES = %w[failed pending running manual scheduled canceled success skipped created].freeze
   STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
-                    failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+                    failed: 4, canceled: 5, skipped: 6, manual: 7,
+                    scheduled: 8 }.freeze
 
   UnknownStatusError = Class.new(StandardError)
 
@@ -24,6 +25,7 @@ module HasStatus
       created = scope_relevant.created.select('count(*)').to_sql
       success = scope_relevant.success.select('count(*)').to_sql
       manual = scope_relevant.manual.select('count(*)').to_sql
+      scheduled = scope_relevant.scheduled.select('count(*)').to_sql
       pending = scope_relevant.pending.select('count(*)').to_sql
       running = scope_relevant.running.select('count(*)').to_sql
       skipped = scope_relevant.skipped.select('count(*)').to_sql
@@ -40,6 +42,7 @@ module HasStatus
         WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending'
         WHEN (#{running})+(#{pending})>0 THEN 'running'
         WHEN (#{manual})>0 THEN 'manual'
+        WHEN (#{scheduled})>0 THEN 'scheduled'
         WHEN (#{created})>0 THEN 'running'
         ELSE 'failed'
       END)"
@@ -74,6 +77,7 @@ module HasStatus
       state :canceled, value: 'canceled'
       state :skipped, value: 'skipped'
       state :manual, value: 'manual'
+      state :scheduled, value: 'scheduled'
     end
 
     scope :created, -> { where(status: 'created') }
@@ -85,6 +89,7 @@ module HasStatus
     scope :canceled, -> { where(status: 'canceled')  }
     scope :skipped, -> { where(status: 'skipped')  }
     scope :manual, -> { where(status: 'manual')  }
+    scope :scheduled, -> { where(status: 'scheduled')  }
     scope :alive, -> { where(status: [:created, :pending, :running]) }
     scope :created_or_pending, -> { where(status: [:created, :pending]) }
     scope :running_or_pending, -> { where(status: [:running, :pending]) }
@@ -92,7 +97,7 @@ module HasStatus
     scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) }
 
     scope :cancelable, -> do
-      where(status: [:running, :pending, :created])
+      where(status: [:running, :pending, :created, :scheduled])
     end
   end
 
@@ -109,7 +114,7 @@ module HasStatus
   end
 
   def blocked?
-    BLOCKED_STATUS == status
+    BLOCKED_STATUS.include?(status)
   end
 
   private
diff --git a/app/presenters/ci/build_presenter.rb b/app/presenters/ci/build_presenter.rb
index 5331cdf632b..33056a809b7 100644
--- a/app/presenters/ci/build_presenter.rb
+++ b/app/presenters/ci/build_presenter.rb
@@ -35,6 +35,10 @@ module Ci
       "#{subject.name} - #{detailed_status.status_tooltip}"
     end
 
+    def execute_in
+      scheduled? && scheduled_at && [0, scheduled_at - Time.now].max
+    end
+
     private
 
     def tooltip_for_badge
diff --git a/app/presenters/commit_status_presenter.rb b/app/presenters/commit_status_presenter.rb
index 65e77ea3f92..29eaad759bb 100644
--- a/app/presenters/commit_status_presenter.rb
+++ b/app/presenters/commit_status_presenter.rb
@@ -8,7 +8,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
     stuck_or_timeout_failure: 'There has been a timeout failure or the job got stuck. Check your timeout limits or try again',
     runner_system_failure: 'There has been a runner system failure, please try again',
     missing_dependency_failure: 'There has been a missing dependency failure',
-    runner_unsupported: 'Your runner is outdated, please upgrade your runner'
+    runner_unsupported: 'Your runner is outdated, please upgrade your runner',
+    stale_schedule: 'Delayed job could not be executed by some reason, please try again'
   }.freeze
 
   private_constant :CALLOUT_FAILURE_MESSAGES
diff --git a/app/serializers/build_action_entity.rb b/app/serializers/build_action_entity.rb
index f9da3f63911..0db7875aa87 100644
--- a/app/serializers/build_action_entity.rb
+++ b/app/serializers/build_action_entity.rb
@@ -12,6 +12,11 @@ class BuildActionEntity < Grape::Entity
   end
 
   expose :playable?, as: :playable
+  expose :scheduled_at, if: -> (build) { build.scheduled? }
+
+  expose :unschedule_path, if: -> (build) { build.scheduled? } do |build|
+    unschedule_project_job_path(build.project, build)
+  end
 
   private
 
diff --git a/app/serializers/job_entity.rb b/app/serializers/job_entity.rb
index 26b29993fec..0b19cb16955 100644
--- a/app/serializers/job_entity.rb
+++ b/app/serializers/job_entity.rb
@@ -24,7 +24,12 @@ class JobEntity < Grape::Entity
     path_to(:play_namespace_project_job, build)
   end
 
+  expose :unschedule_path, if: -> (*) { scheduled? } do |build|
+    path_to(:unschedule_namespace_project_job, build)
+  end
+
   expose :playable?, as: :playable
+  expose :scheduled_at, if: -> (*) { scheduled? }
   expose :created_at
   expose :updated_at
   expose :detailed_status, as: :status, with: DetailedStatusEntity
@@ -47,6 +52,10 @@ class JobEntity < Grape::Entity
     build.playable? && can?(request.current_user, :update_build, build)
   end
 
+  def scheduled?
+    build.scheduled?
+  end
+
   def detailed_status
     build.detailed_status(request.current_user)
   end
diff --git a/app/serializers/pipeline_details_entity.rb b/app/serializers/pipeline_details_entity.rb
index 3b56767f774..d78ad4af4dc 100644
--- a/app/serializers/pipeline_details_entity.rb
+++ b/app/serializers/pipeline_details_entity.rb
@@ -5,5 +5,6 @@ class PipelineDetailsEntity < PipelineEntity
     expose :ordered_stages, as: :stages, using: StageEntity
     expose :artifacts, using: BuildArtifactEntity
     expose :manual_actions, using: BuildActionEntity
+    expose :scheduled_actions, using: BuildActionEntity
   end
 end
diff --git a/app/serializers/pipeline_serializer.rb b/app/serializers/pipeline_serializer.rb
index 4f31af3c46d..7451433a841 100644
--- a/app/serializers/pipeline_serializer.rb
+++ b/app/serializers/pipeline_serializer.rb
@@ -13,6 +13,7 @@ class PipelineSerializer < BaseSerializer
         :cancelable_statuses,
         :trigger_requests,
         :manual_actions,
+        :scheduled_actions,
         :artifacts,
         {
           pending_builds: :project,
diff --git a/app/services/ci/enqueue_build_service.rb b/app/services/ci/enqueue_build_service.rb
deleted file mode 100644
index 8140651d980..00000000000
--- a/app/services/ci/enqueue_build_service.rb
+++ /dev/null
@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-module Ci
-  class EnqueueBuildService < BaseService
-    def execute(build)
-      build.enqueue
-    end
-  end
-end
diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb
new file mode 100644
index 00000000000..d9f8e7cb452
--- /dev/null
+++ b/app/services/ci/process_build_service.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Ci
+  class ProcessBuildService < BaseService
+    def execute(build, current_status)
+      if valid_statuses_for_when(build.when).include?(current_status)
+        if build.schedulable?
+          build.schedule
+        elsif build.action?
+          build.actionize
+        else
+          enqueue(build)
+        end
+
+        true
+      else
+        build.skip
+        false
+      end
+    end
+
+    private
+
+    def enqueue(build)
+      build.enqueue
+    end
+
+    def valid_statuses_for_when(value)
+      case value
+      when 'on_success'
+        %w[success skipped]
+      when 'on_failure'
+        %w[failed]
+      when 'always'
+        %w[success failed skipped]
+      when 'manual'
+        %w[success skipped]
+      when 'delayed'
+        %w[success skipped]
+      else
+        []
+      end
+    end
+  end
+end
diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 69341a6c263..446188347df 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -24,42 +24,18 @@ module Ci
     def process_stage(index)
       current_status = status_for_prior_stages(index)
 
-      return if HasStatus::BLOCKED_STATUS == current_status
+      return if HasStatus::BLOCKED_STATUS.include?(current_status)
 
       if HasStatus::COMPLETED_STATUSES.include?(current_status)
         created_builds_in_stage(index).select do |build|
           Gitlab::OptimisticLocking.retry_lock(build) do |subject|
-            process_build(subject, current_status)
+            Ci::ProcessBuildService.new(project, @user)
+              .execute(build, current_status)
           end
         end
       end
     end
 
-    def process_build(build, current_status)
-      if valid_statuses_for_when(build.when).include?(current_status)
-        build.action? ? build.actionize : enqueue_build(build)
-        true
-      else
-        build.skip
-        false
-      end
-    end
-
-    def valid_statuses_for_when(value)
-      case value
-      when 'on_success'
-        %w[success skipped]
-      when 'on_failure'
-        %w[failed]
-      when 'always'
-        %w[success failed skipped]
-      when 'manual'
-        %w[success skipped]
-      else
-        []
-      end
-    end
-
     # rubocop: disable CodeReuse/ActiveRecord
     def status_for_prior_stages(index)
       pipeline.builds.where('stage_idx < ?', index).latest.status || 'success'
@@ -101,9 +77,5 @@ module Ci
         .update_all(retried: true) if latest_statuses.any?
     end
     # rubocop: enable CodeReuse/ActiveRecord
-
-    def enqueue_build(build)
-      Ci::EnqueueBuildService.new(project, @user).execute(build)
-    end
   end
 end
diff --git a/app/services/ci/run_scheduled_build_service.rb b/app/services/ci/run_scheduled_build_service.rb
new file mode 100644
index 00000000000..8e4a628296f
--- /dev/null
+++ b/app/services/ci/run_scheduled_build_service.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Ci
+  class RunScheduledBuildService < ::BaseService
+    def execute(build)
+      unless can?(current_user, :update_build, build)
+        raise Gitlab::Access::AccessDeniedError
+      end
+
+      build.enqueue_scheduled!
+    end
+  end
+end
diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml
index 44c1453e239..59c297c46a5 100644
--- a/app/views/projects/ci/builds/_build.html.haml
+++ b/app/views/projects/ci/builds/_build.html.haml
@@ -47,7 +47,9 @@
         %span.badge.badge-info triggered
       - if job.try(:allow_failure)
         %span.badge.badge-danger allowed to fail
-      - if job.action?
+      - if job.schedulable?
+        %span.badge.badge-info= s_('DelayedJobs|scheduled')
+      - elsif job.action?
         %span.badge.badge-info manual
 
   - if pipeline_link
@@ -101,6 +103,24 @@
         - if job.active?
           = link_to cancel_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: 'Cancel', class: 'btn btn-build' do
             = icon('remove', class: 'cred')
+        - elsif job.scheduled?
+          .btn-group
+            .btn.btn-default.has-tooltip{ disabled: true,
+              title: job.scheduled_at }
+              = sprite_icon('planning')
+              = duration_in_numbers(job.execute_in, true)
+            - confirmation_message = s_("DelayedJobs|Are you sure you want to run %{job_name} immediately? This job will run automatically after it's timer finishes.") % { job_name: job.name }
+            = link_to play_project_job_path(job.project, job, return_to: request.original_url),
+              method: :post,
+              title: s_('DelayedJobs|Start now'),
+              class: 'btn btn-default btn-build has-tooltip',
+              data: { confirm: confirmation_message } do
+              = sprite_icon('play')
+            = link_to unschedule_project_job_path(job.project, job, return_to: request.original_url),
+              method: :post,
+              title: s_('DelayedJobs|Unschedule'),
+              class: 'btn btn-default btn-build has-tooltip' do
+              = sprite_icon('time-out')
         - elsif allow_retry
           - if job.playable? && !admin && can?(current_user, :update_build, job)
             = link_to play_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: 'Play', class: 'btn btn-build' do
diff --git a/app/views/shared/icons/_icon_status_scheduled.svg b/app/views/shared/icons/_icon_status_scheduled.svg
new file mode 100644
index 00000000000..ca6e4efce50
--- /dev/null
+++ b/app/views/shared/icons/_icon_status_scheduled.svg
@@ -0,0 +1 @@
+<svg width="14" height="14" viewBox="0 0 14 14" xmlns="http://www.w3.org/2000/svg"><circle cx="7" cy="7" r="7"/><circle fill="#FFF" cx="7" cy="7" r="6"/><g transform="translate(2.75 2.75)" fill-rule="nonzero"><path d="M4.165 7.81a3.644 3.644 0 1 1 0-7.29 3.644 3.644 0 0 1 0 7.29zm0-1.042a2.603 2.603 0 1 0 0-5.206 2.603 2.603 0 0 0 0 5.206z"/><rect x="3.644" y="2.083" width="1.041" height="2.603" rx=".488"/><rect x="3.644" y="3.644" width="2.083" height="1.041" rx=".488"/></g></svg>
\ No newline at end of file
diff --git a/app/views/shared/icons/_icon_status_scheduled_borderless.svg b/app/views/shared/icons/_icon_status_scheduled_borderless.svg
new file mode 100644
index 00000000000..dc38c01d898
--- /dev/null
+++ b/app/views/shared/icons/_icon_status_scheduled_borderless.svg
@@ -0,0 +1 @@
+<svg width="22px" height="22px" viewBox="0 0 22 22" version="1.1" xmlns="http://www.w3.org/2000/svg"><path d="M6.16 11.55a5.39 5.39 0 1 1 0-10.78 5.39 5.39 0 0 1 0 10.78zm0-1.54a3.85 3.85 0 1 0 0-7.7 3.85 3.85 0 0 0 0 7.7z"/><rect x="5.39" y="3.08" width="1.54" height="3.85" rx=".767"/><rect x="5.39" y="5.39" width="3.08" height="1.54" rx=".767"/></svg>
\ No newline at end of file
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 1eeb972cee9..f21789de37d 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -70,6 +70,7 @@
 - pipeline_processing:pipeline_update
 - pipeline_processing:stage_update
 - pipeline_processing:update_head_pipeline_for_merge_request
+- pipeline_processing:ci_build_schedule
 
 - repository_check:repository_check_clear
 - repository_check:repository_check_batch
diff --git a/app/workers/ci/build_schedule_worker.rb b/app/workers/ci/build_schedule_worker.rb
new file mode 100644
index 00000000000..da219adffc6
--- /dev/null
+++ b/app/workers/ci/build_schedule_worker.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Ci
+  class BuildScheduleWorker
+    include ApplicationWorker
+    include PipelineQueue
+
+    queue_namespace :pipeline_processing
+
+    def perform(build_id)
+      ::Ci::Build.find_by_id(build_id).try do |build|
+        break unless build.scheduled?
+
+        Ci::RunScheduledBuildService
+          .new(build.project, build.user).execute(build)
+      end
+    end
+  end
+end
diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb
index f6bca1176d1..25809f68080 100644
--- a/app/workers/stuck_ci_jobs_worker.rb
+++ b/app/workers/stuck_ci_jobs_worker.rb
@@ -8,6 +8,7 @@ class StuckCiJobsWorker
 
   BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
   BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
+  BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
   BUILD_PENDING_STUCK_TIMEOUT = 1.hour
 
   def perform
@@ -15,9 +16,10 @@ class StuckCiJobsWorker
 
     Rails.logger.info "#{self.class}: Cleaning stuck builds"
 
-    drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT
-    drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT
-    drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT
+    drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
+    drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
+    drop :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, 'scheduled_at IS NOT NULL AND scheduled_at < ?', :stale_schedule
+    drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
 
     remove_lease
   end
@@ -32,25 +34,25 @@ class StuckCiJobsWorker
     Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
   end
 
-  def drop(status, timeout)
-    search(status, timeout) do |build|
-      drop_build :outdated, build, status, timeout
+  def drop(status, timeout, condition, reason)
+    search(status, timeout, condition) do |build|
+      drop_build :outdated, build, status, timeout, reason
     end
   end
 
-  def drop_stuck(status, timeout)
-    search(status, timeout) do |build|
+  def drop_stuck(status, timeout, condition, reason)
+    search(status, timeout, condition) do |build|
       break unless build.stuck?
 
-      drop_build :stuck, build, status, timeout
+      drop_build :stuck, build, status, timeout, reason
     end
   end
 
   # rubocop: disable CodeReuse/ActiveRecord
-  def search(status, timeout)
+  def search(status, timeout, condition)
     loop do
       jobs = Ci::Build.where(status: status)
-        .where('ci_builds.updated_at < ?', timeout.ago)
+        .where(condition, timeout.ago)
         .includes(:tags, :runner, project: :namespace)
         .limit(100)
         .to_a
@@ -63,10 +65,10 @@ class StuckCiJobsWorker
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
-  def drop_build(type, build, status, timeout)
-    Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})"
+  def drop_build(type, build, status, timeout, reason)
+    Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})"
     Gitlab::OptimisticLocking.retry_lock(build, 3) do |b|
-      b.drop(:stuck_or_timeout_failure)
+      b.drop(reason)
     end
   end
 end