Merge branch 'live-trace-v2' into live-trace-v2-efficient-destroy-all

13 files changed, 303 insertions, 73 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 862933bf127..a734cc7a26b 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -220,12 +220,15 @@ class ApplicationSetting < ActiveRecord::Base
     end
   end
 
+  validate :terms_exist, if: :enforce_terms?
+
   before_validation :ensure_uuid!
 
   before_save :ensure_runners_registration_token
   before_save :ensure_health_check_access_token
 
   after_commit do
+    reset_memoized_terms
     Rails.cache.write(CACHE_KEY, self)
   end
 
@@ -507,6 +510,16 @@ class ApplicationSetting < ActiveRecord::Base
     password_authentication_enabled_for_web? || password_authentication_enabled_for_git?
   end
 
+  delegate :terms, to: :latest_terms, allow_nil: true
+  def latest_terms
+    @latest_terms ||= Term.latest
+  end
+
+  def reset_memoized_terms
+    @latest_terms = nil
+    latest_terms
+  end
+
   private
 
   def ensure_uuid!
@@ -520,4 +533,10 @@ class ApplicationSetting < ActiveRecord::Base
     errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless
       invalid.empty?
   end
+
+  def terms_exist
+    return unless enforce_terms?
+
+    errors.add(:terms, "You need to set terms to be enforced") unless terms.present?
+  end
 end
diff --git a/app/models/application_setting/term.rb b/app/models/application_setting/term.rb
new file mode 100644
index 00000000000..e8ce0ccbb71
--- /dev/null
+++ b/app/models/application_setting/term.rb
@@ -0,0 +1,13 @@
+class ApplicationSetting
+  class Term < ActiveRecord::Base
+    include CacheMarkdownField
+
+    validates :terms, presence: true
+
+    cache_markdown_field :terms
+
+    def self.latest
+      order(:id).last
+    end
+  end
+end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index e1b9bc76475..c184f398f6c 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -32,6 +32,8 @@ module Ci
     has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
     has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
 
+    accepts_nested_attributes_for :variables, reject_if: :persisted?
+
     delegate :id, to: :project, prefix: true
     delegate :full_path, to: :project, prefix: true
 
diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb
index de5aae17a15..38e14ffbc0c 100644
--- a/app/models/ci/pipeline_variable.rb
+++ b/app/models/ci/pipeline_variable.rb
@@ -5,6 +5,8 @@ module Ci
 
     belongs_to :pipeline
 
+    alias_attribute :secret_value, :value
+
     validates :key, uniqueness: { scope: :pipeline_id }
   end
 end
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 5a4c56ec0dc..23078f1c3ed 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -14,31 +14,49 @@ module Ci
     has_many :builds
     has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_many :projects, through: :runner_projects
+    has_many :runner_namespaces
+    has_many :groups, through: :runner_namespaces
 
     has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build'
 
     before_validation :set_default_values
 
-    scope :specific, ->() { where(is_shared: false) }
-    scope :shared, ->() { where(is_shared: true) }
-    scope :active, ->() { where(active: true) }
-    scope :paused, ->() { where(active: false) }
-    scope :online, ->() { where('contacted_at > ?', contact_time_deadline) }
-    scope :ordered, ->() { order(id: :desc) }
+    scope :specific, -> { where(is_shared: false) }
+    scope :shared, -> { where(is_shared: true) }
+    scope :active, -> { where(active: true) }
+    scope :paused, -> { where(active: false) }
+    scope :online, -> { where('contacted_at > ?', contact_time_deadline) }
+    scope :ordered, -> { order(id: :desc) }
 
-    scope :owned_or_shared, ->(project_id) do
-      joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id')
-        .where("ci_runner_projects.project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id)
+    scope :belonging_to_project, -> (project_id) {
+      joins(:runner_projects).where(ci_runner_projects: { project_id: project_id })
+    }
+
+    scope :belonging_to_parent_group_of_project, -> (project_id) {
+      project_groups = ::Group.joins(:projects).where(projects: { id: project_id })
+      hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors
+
+      joins(:groups).where(namespaces: { id: hierarchy_groups })
+    }
+
+    scope :owned_or_shared, -> (project_id) do
+      union = Gitlab::SQL::Union.new(
+        [belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared],
+        remove_duplicates: false
+      )
+      from("(#{union.to_sql}) ci_runners")
     end
 
     scope :assignable_for, ->(project) do
       # FIXME: That `to_sql` is needed to workaround a weird Rails bug.
       #        Without that, placeholders would miss one and couldn't match.
       where(locked: false)
-        .where.not("id IN (#{project.runners.select(:id).to_sql})").specific
+        .where.not("ci_runners.id IN (#{project.runners.select(:id).to_sql})")
+        .specific
     end
 
     validate :tag_constraints
+    validate :either_projects_or_group
     validates :access_level, presence: true
 
     acts_as_taggable
@@ -50,6 +68,12 @@ module Ci
       ref_protected: 1
     }
 
+    enum runner_type: {
+      instance_type: 1,
+      group_type: 2,
+      project_type: 3
+    }
+
     cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address
 
     chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout
@@ -120,6 +144,14 @@ module Ci
       !shared?
     end
 
+    def assigned_to_group?
+      runner_namespaces.any?
+    end
+
+    def assigned_to_project?
+      runner_projects.any?
+    end
+
     def can_pick?(build)
       return false if self.ref_protected? && !build.protected?
 
@@ -174,6 +206,12 @@ module Ci
       end
     end
 
+    def pick_build!(build)
+      if can_pick?(build)
+        tick_runner_queue
+      end
+    end
+
     private
 
     def cleanup_runner_queue
@@ -205,7 +243,17 @@ module Ci
     end
 
     def assignable_for?(project_id)
-      is_shared? || projects.exists?(id: project_id)
+      self.class.owned_or_shared(project_id).where(id: self.id).any?
+    end
+
+    def either_projects_or_group
+      if groups.many?
+        errors.add(:runner, 'can only be assigned to one group')
+      end
+
+      if assigned_to_group? && assigned_to_project?
+        errors.add(:runner, 'can only be assigned either to projects or to a group')
+      end
     end
 
     def accepting_tags?(build)
diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb
new file mode 100644
index 00000000000..3269f86e8ca
--- /dev/null
+++ b/app/models/ci/runner_namespace.rb
@@ -0,0 +1,9 @@
+module Ci
+  class RunnerNamespace < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :runner
+    belongs_to :namespace, class_name: '::Namespace'
+    belongs_to :group, class_name: '::Group', foreign_key: :namespace_id
+  end
+end
diff --git a/app/models/group.rb b/app/models/group.rb
index 9b42bbf99be..f493836a92e 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -9,6 +9,7 @@ class Group < Namespace
   include SelectForProjectAuthorization
   include LoadedInGroupList
   include GroupDescendant
+  include TokenAuthenticatable
 
   has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
   alias_method :members, :group_members
@@ -43,6 +44,8 @@ class Group < Namespace
 
   validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 }
 
+  add_authentication_token_field :runners_token
+
   after_create :post_create_hook
   after_destroy :post_destroy_hook
   after_save :update_two_factor_requirement
@@ -294,6 +297,13 @@ class Group < Namespace
     refresh_members_authorized_projects(blocking: false)
   end
 
+  # each existing group needs to have a `runners_token`.
+  # we do this on read since migrating all existing groups is not a feasible
+  # solution.
+  def runners_token
+    ensure_runners_token!
+  end
+
   private
 
   def update_two_factor_requirement
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index c29a53e5ce7..5621eeba7c4 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -21,6 +21,9 @@ class Namespace < ActiveRecord::Base
   has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :project_statistics
 
+  has_many :runner_namespaces, class_name: 'Ci::RunnerNamespace'
+  has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner'
+
   # This should _not_ be `inverse_of: :namespace`, because that would also set
   # `user.namespace` when this user creates a group with themselves as `owner`.
   belongs_to :owner, class_name: "User"
diff --git a/app/models/project.rb b/app/models/project.rb
index e48ecea1053..37f1fcea280 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -68,6 +68,9 @@ class Project < ActiveRecord::Base
   before_save :ensure_runners_token
 
   after_save :update_project_statistics, if: :namespace_id_changed?
+
+  after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? }
+
   after_create :create_project_feature, unless: :project_feature
 
   after_create :create_ci_cd_settings,
@@ -161,6 +164,8 @@ class Project < ActiveRecord::Base
   has_one :fork_network_member
   has_one :fork_network, through: :fork_network_member
 
+  has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
+
   # Merge Requests for target project should be removed with it
   has_many :merge_requests, foreign_key: 'target_project_id'
   has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest'
@@ -235,13 +240,11 @@ class Project < ActiveRecord::Base
   has_many :project_deploy_tokens
   has_many :deploy_tokens, through: :project_deploy_tokens
 
-  has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
-
   has_one :auto_devops, class_name: 'ProjectAutoDevops'
   has_many :custom_attributes, class_name: 'ProjectCustomAttribute'
 
   has_many :project_badges, class_name: 'ProjectBadge'
-  has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting'
+  has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
@@ -252,6 +255,7 @@ class Project < ActiveRecord::Base
   delegate :members, to: :team, prefix: true
   delegate :add_user, :add_users, to: :team
   delegate :add_guest, :add_reporter, :add_developer, :add_master, :add_role, to: :team
+  delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :ci_cd_settings
 
   # Validations
   validates :creator, presence: true, on: :create
@@ -337,6 +341,11 @@ class Project < ActiveRecord::Base
   scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
   scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
 
+  scope :with_group_runners_enabled, -> do
+    joins(:ci_cd_settings)
+    .where(project_ci_cd_settings: { group_runners_enabled: true })
+  end
+
   enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 }
 
   chronic_duration_attr :build_timeout_human_readable, :build_timeout, default: 3600
@@ -386,55 +395,9 @@ class Project < ActiveRecord::Base
   scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 
   scope :excluding_project, ->(project) { where.not(id: project) }
-  scope :import_started, -> { where(import_status: 'started') }
 
-  state_machine :import_status, initial: :none do
-    event :import_schedule do
-      transition [:none, :finished, :failed] => :scheduled
-    end
-
-    event :force_import_start do
-      transition [:none, :finished, :failed] => :started
-    end
-
-    event :import_start do
-      transition scheduled: :started
-    end
-
-    event :import_finish do
-      transition started: :finished
-    end
-
-    event :import_fail do
-      transition [:scheduled, :started] => :failed
-    end
-
-    event :import_retry do
-      transition failed: :started
-    end
-
-    state :scheduled
-    state :started
-    state :finished
-    state :failed
-
-    after_transition [:none, :finished, :failed] => :scheduled do |project, _|
-      project.run_after_commit do
-        job_id = add_import_job
-        update(import_jid: job_id) if job_id
-      end
-    end
-
-    after_transition started: :finished do |project, _|
-      project.reset_cache_and_import_attrs
-
-      if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists?
-        project.run_after_commit do
-          Projects::AfterImportService.new(project).execute
-        end
-      end
-    end
-  end
+  scope :joins_import_state, -> { joins("LEFT JOIN project_mirror_data import_state ON import_state.project_id = projects.id") }
+  scope :import_started, -> { joins_import_state.where("import_state.status = 'started' OR projects.import_status = 'started'") }
 
   class << self
     # Searches for a list of projects based on the query given in `query`.
@@ -664,10 +627,6 @@ class Project < ActiveRecord::Base
     external_import? || forked? || gitlab_project_import? || bare_repository_import?
   end
 
-  def no_import?
-    import_status == 'none'
-  end
-
   def external_import?
     import_url.present?
   end
@@ -680,6 +639,93 @@ class Project < ActiveRecord::Base
     import_started? || import_scheduled?
   end
 
+  def import_state_args
+    {
+      status: self[:import_status],
+      jid: self[:import_jid],
+      last_error: self[:import_error]
+    }
+  end
+
+  def ensure_import_state
+    return if self[:import_status] == 'none' || self[:import_status].nil?
+    return unless import_state.nil?
+
+    create_import_state(import_state_args)
+
+    update_column(:import_status, 'none')
+  end
+
+  def import_schedule
+    ensure_import_state
+
+    import_state&.schedule
+  end
+
+  def force_import_start
+    ensure_import_state
+
+    import_state&.force_start
+  end
+
+  def import_start
+    ensure_import_state
+
+    import_state&.start
+  end
+
+  def import_fail
+    ensure_import_state
+
+    import_state&.fail_op
+  end
+
+  def import_finish
+    ensure_import_state
+
+    import_state&.finish
+  end
+
+  def import_jid=(new_jid)
+    ensure_import_state
+
+    import_state&.jid = new_jid
+  end
+
+  def import_jid
+    ensure_import_state
+
+    import_state&.jid
+  end
+
+  def import_error=(new_error)
+    ensure_import_state
+
+    import_state&.last_error = new_error
+  end
+
+  def import_error
+    ensure_import_state
+
+    import_state&.last_error
+  end
+
+  def import_status=(new_status)
+    ensure_import_state
+
+    import_state&.status = new_status
+  end
+
+  def import_status
+    ensure_import_state
+
+    import_state&.status || 'none'
+  end
+
+  def no_import?
+    import_status == 'none'
+  end
+
   def import_started?
     # import? does SQL work so only run it if it looks like there's an import running
     import_status == 'started' && import?
@@ -1306,12 +1352,17 @@ class Project < ActiveRecord::Base
     @shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none
   end
 
-  def active_shared_runners
-    @active_shared_runners ||= shared_runners.active
+  def group_runners
+    @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_group_of_project(self.id) : Ci::Runner.none
+  end
+
+  def all_runners
+    union = Gitlab::SQL::Union.new([runners, group_runners, shared_runners])
+    Ci::Runner.from("(#{union.to_sql}) ci_runners")
   end
 
   def any_runners?(&block)
-    active_runners.any?(&block) || active_shared_runners.any?(&block)
+    all_runners.active.any?(&block)
   end
 
   def valid_runners_token?(token)
@@ -1476,7 +1527,7 @@ class Project < ActiveRecord::Base
   def rename_repo_notify!
     # When we import a project overwriting the original project, there
     # is a move operation. In that case we don't want to send the instructions.
-    send_move_instructions(full_path_was) unless started?
+    send_move_instructions(full_path_was) unless import_started?
     expires_full_path_cache
 
     self.old_path_with_namespace = full_path_was
@@ -1530,7 +1581,8 @@ class Project < ActiveRecord::Base
     return unless import_jid
 
     Gitlab::SidekiqStatus.unset(import_jid)
-    update_column(:import_jid, nil)
+
+    import_state.update_column(:jid, nil)
   end
 
   def running_or_pending_build_count(force: false)
@@ -1549,7 +1601,8 @@ class Project < ActiveRecord::Base
     sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message)
 
     import_fail
-    update_column(:import_error, sanitized_message)
+
+    import_state.update_column(:last_error, sanitized_message)
   rescue ActiveRecord::ActiveRecordError => e
     Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}")
   ensure
@@ -1879,6 +1932,10 @@ class Project < ActiveRecord::Base
     []
   end
 
+  def toggle_ci_cd_settings!(settings_attribute)
+    ci_cd_settings.toggle!(settings_attribute)
+  end
+
   def gitlab_deploy_token
     @gitlab_deploy_token ||= deploy_tokens.gitlab_deploy_token
   end
diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb
index 9f10a93148c..588cced5781 100644
--- a/app/models/project_ci_cd_setting.rb
+++ b/app/models/project_ci_cd_setting.rb
@@ -1,5 +1,5 @@
 class ProjectCiCdSetting < ActiveRecord::Base
-  belongs_to :project
+  belongs_to :project, inverse_of: :ci_cd_settings
 
   # The version of the schema that first introduced this model/table.
   MINIMUM_SCHEMA_VERSION = 20180403035759
diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb
new file mode 100644
index 00000000000..1605317ae14
--- /dev/null
+++ b/app/models/project_import_state.rb
@@ -0,0 +1,55 @@
+class ProjectImportState < ActiveRecord::Base
+  include AfterCommitQueue
+
+  self.table_name = "project_mirror_data"
+
+  belongs_to :project, inverse_of: :import_state
+
+  validates :project, presence: true
+
+  state_machine :status, initial: :none do
+    event :schedule do
+      transition [:none, :finished, :failed] => :scheduled
+    end
+
+    event :force_start do
+      transition [:none, :finished, :failed] => :started
+    end
+
+    event :start do
+      transition scheduled: :started
+    end
+
+    event :finish do
+      transition started: :finished
+    end
+
+    event :fail_op do
+      transition [:scheduled, :started] => :failed
+    end
+
+    state :scheduled
+    state :started
+    state :finished
+    state :failed
+
+    after_transition [:none, :finished, :failed] => :scheduled do |state, _|
+      state.run_after_commit do
+        job_id = project.add_import_job
+        update(jid: job_id) if job_id
+      end
+    end
+
+    after_transition started: :finished do |state, _|
+      project = state.project
+
+      project.reset_cache_and_import_attrs
+
+      if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists?
+        state.run_after_commit do
+          Projects::AfterImportService.new(project).execute
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/term_agreement.rb b/app/models/term_agreement.rb
new file mode 100644
index 00000000000..8458a231bbd
--- /dev/null
+++ b/app/models/term_agreement.rb
@@ -0,0 +1,6 @@
+class TermAgreement < ActiveRecord::Base
+  belongs_to :term, class_name: 'ApplicationSetting::Term'
+  belongs_to :user
+
+  validates :user, :term, presence: true
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index 4a602ffbb05..a9cfd39f604 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -138,6 +138,8 @@ class User < ActiveRecord::Base
   has_many :custom_attributes, class_name: 'UserCustomAttribute'
   has_many :callouts, class_name: 'UserCallout'
   has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_many :term_agreements
+  belongs_to :accepted_term, class_name: 'ApplicationSetting::Term'
 
   #
   # Validations
@@ -1187,6 +1189,10 @@ class User < ActiveRecord::Base
     max_member_access_for_group_ids([group_id])[group_id]
   end
 
+  def terms_accepted?
+    accepted_term_id.present?
+  end
+
   protected
 
   # override, from Devise::Validatable