20 files changed, 296 insertions, 165 deletions

diff --git a/app/models/ci/application_record.rb b/app/models/ci/application_record.rb
index ea7b1104e36..52f02bfb2fd 100644
--- a/app/models/ci/application_record.rb
+++ b/app/models/ci/application_record.rb
@@ -13,7 +13,7 @@ module Ci
     end
 
     def self.model_name
-      @model_name ||= ActiveModel::Name.new(self, nil, self.name.demodulize)
+      @model_name ||= ActiveModel::Name.new(self, nil, name.demodulize)
     end
   end
 end
diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index 4af31fd37f2..697f06fbffd 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -55,7 +55,11 @@ module Ci
     end
 
     def retryable?
-      false
+      return false unless Feature.enabled?(:ci_recreate_downstream_pipeline, project)
+
+      return false if failed? && (pipeline_loop_detected? || reached_max_descendant_pipelines_depth?)
+
+      super
     end
 
     def self.with_preloads
@@ -76,9 +80,9 @@ module Ci
     def inherit_status_from_downstream!(pipeline)
       case pipeline.status
       when 'success'
-        self.success!
+        success!
       when 'failed', 'canceled', 'skipped'
-        self.drop!
+        drop!
       else
         false
       end
@@ -186,6 +190,10 @@ module Ci
     def persisted_environment
     end
 
+    def deployment_job?
+      false
+    end
+
     def execute_hooks
       raise NotImplementedError
     end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 0139b025d98..f8b3777841d 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -34,11 +34,11 @@ module Ci
     DEPLOYMENT_NAMES = %w[deploy release rollout].freeze
 
     has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable
-    has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build
+    has_one :pending_state, class_name: 'Ci::BuildPendingState', foreign_key: :build_id, inverse_of: :build
     has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id
     has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build
-    has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build
+    has_many :report_results, class_name: 'Ci::BuildReportResult', foreign_key: :build_id, inverse_of: :build
     has_one :namespace, through: :project
 
     # Projects::DestroyService destroys Ci::Pipelines, which use_fast_destroy on :job_artifacts
@@ -49,16 +49,18 @@ module Ci
     has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id, inverse_of: :job
     has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id
 
-    has_many :pages_deployments, inverse_of: :ci_build
+    has_many :pages_deployments, foreign_key: :ci_build_id, inverse_of: :ci_build
 
     Ci::JobArtifact.file_types.each do |key, value|
-      has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+      has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', foreign_key: :job_id, inverse_of: :job
     end
 
-    has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
-    has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build
+    has_one :runner_machine, through: :metadata, class_name: 'Ci::RunnerMachine'
 
-    has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', inverse_of: :build, foreign_key: :ci_build_id
+    has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, foreign_key: :build_id, inverse_of: :build
+    has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', foreign_key: :build_id, inverse_of: :build
+
+    has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', foreign_key: :ci_build_id, inverse_of: :build
 
     accepts_nested_attributes_for :runner_session, update_only: true
     accepts_nested_attributes_for :job_variables
@@ -88,6 +90,12 @@ module Ci
 
     scope :unstarted, -> { where(runner_id: nil) }
 
+    scope :with_any_artifacts, -> do
+      where('EXISTS (?)',
+        Ci::JobArtifact.select(1).where("#{Ci::Build.quoted_table_name}.id = #{Ci::JobArtifact.quoted_table_name}.job_id")
+      )
+    end
+
     scope :with_downloadable_artifacts, -> do
       where('EXISTS (?)',
         Ci::JobArtifact.select(1)
@@ -179,6 +187,8 @@ module Ci
       run_after_commit { build.execute_hooks }
     end
 
+    after_commit :track_ci_secrets_management_id_tokens_usage, on: :create, if: :id_tokens?
+
     class << self
       # This is needed for url_for to work,
       # as the controller is JobsController
@@ -382,21 +392,21 @@ module Ci
 
     def detailed_status(current_user)
       Gitlab::Ci::Status::Build::Factory
-        .new(self.present, current_user)
+        .new(present, current_user)
         .fabricate!
     end
 
     def other_manual_actions
-      pipeline.manual_actions.reject { |action| action.name == self.name }
+      pipeline.manual_actions.reject { |action| action.name == name }
     end
 
     def other_scheduled_actions
-      pipeline.scheduled_actions.reject { |action| action.name == self.name }
+      pipeline.scheduled_actions.reject { |action| action.name == name }
     end
 
     def pages_generator?
       Gitlab.config.pages.enabled &&
-        self.name == 'pages'
+        name == 'pages'
     end
 
     def runnable?
@@ -452,7 +462,7 @@ module Ci
     end
 
     def retries_count
-      pipeline.builds.retried.where(name: self.name).count
+      pipeline.builds.retried.where(name: name).count
     end
 
     override :all_met_to_become_pending?
@@ -525,19 +535,19 @@ module Ci
     end
 
     def deployment_job?
-      has_environment_keyword? && self.environment_action == 'start'
+      has_environment_keyword? && environment_action == 'start'
     end
 
     def stops_environment?
-      has_environment_keyword? && self.environment_action == 'stop'
+      has_environment_keyword? && environment_action == 'stop'
     end
 
     def environment_action
-      self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options
+      options.fetch(:environment, {}).fetch(:action, 'start') if options
     end
 
     def environment_tier_from_options
-      self.options.dig(:environment, :deployment_tier) if self.options
+      options.dig(:environment, :deployment_tier) if options
     end
 
     def environment_tier
@@ -827,7 +837,7 @@ module Ci
     end
 
     def erased?
-      !self.erased_at.nil?
+      !erased_at.nil?
     end
 
     def artifacts_expired?
@@ -860,14 +870,14 @@ module Ci
     end
 
     def keep_artifacts!
-      self.update(artifacts_expire_at: nil)
-      self.job_artifacts.update_all(expire_at: nil)
+      update(artifacts_expire_at: nil)
+      job_artifacts.update_all(expire_at: nil)
     end
 
-    def artifacts_file_for_type(type)
+    def artifact_for_type(type)
       file_types = Ci::JobArtifact.associated_file_types_for(type)
       file_types_ids = file_types&.map { |file_type| Ci::JobArtifact.file_types[file_type] }
-      job_artifacts.find_by(file_type: file_types_ids)&.file
+      job_artifacts.find_by(file_type: file_types_ids)
     end
 
     def steps
@@ -1092,11 +1102,11 @@ module Ci
     # without actually loading data.
     #
     def all_queuing_entries
-      ::Ci::PendingBuild.where(build_id: self.id)
+      ::Ci::PendingBuild.where(build_id: id)
     end
 
     def all_runtime_metadata
-      ::Ci::RunningBuild.where(build_id: self.id)
+      ::Ci::RunningBuild.where(build_id: id)
     end
 
     def shared_runner_build?
@@ -1281,6 +1291,23 @@ module Ci
           .increment(status: status)
       end
     end
+
+    def track_ci_secrets_management_id_tokens_usage
+      ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('i_ci_secrets_management_id_tokens_build_created', values: user_id)
+
+      Gitlab::Tracking.event(
+        self.class.to_s,
+        'create_id_tokens',
+        namespace: namespace,
+        user: user,
+        label: 'redis_hll_counters.ci_secrets_management.i_ci_secrets_management_id_tokens_build_created_monthly',
+        ultimate_namespace_id: namespace.root_ancestor.id,
+        context: [Gitlab::Tracking::ServicePingContext.new(
+          data_source: :redis_hll,
+          event: 'i_ci_secrets_management_id_tokens_build_created'
+        ).to_context]
+      )
+    end
   end
 end
 
diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb
index 1dcb9190f11..b294afd405d 100644
--- a/app/models/ci/build_metadata.rb
+++ b/app/models/ci/build_metadata.rb
@@ -18,6 +18,7 @@ module Ci
 
     belongs_to :build, class_name: 'CommitStatus'
     belongs_to :project
+    belongs_to :runner_machine, class_name: 'Ci::RunnerMachine'
 
     before_create :set_build_project
 
@@ -67,7 +68,7 @@ module Ci
     private
 
     def set_build_project
-      self.project_id ||= self.build.project_id
+      self.project_id ||= build.project_id
     end
 
     def timeout_with_highest_precedence
diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb
index 3fa17d6d286..03d1bd14bfb 100644
--- a/app/models/ci/build_need.rb
+++ b/app/models/ci/build_need.rb
@@ -4,6 +4,9 @@ module Ci
   class BuildNeed < Ci::ApplicationRecord
     include Ci::Partitionable
     include BulkInsertSafe
+    include IgnorableColumns
+
+    ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-04-22'
 
     belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs
 
diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb
index 20c0b04e228..5773b6132be 100644
--- a/app/models/ci/build_runner_session.rb
+++ b/app/models/ci/build_runner_session.rb
@@ -20,7 +20,7 @@ module Ci
     validates :url, public_url: { schemes: %w(https) }
 
     def terminal_specification
-      wss_url = Gitlab::UrlHelpers.as_wss(Addressable::URI.escape(self.url))
+      wss_url = Gitlab::UrlHelpers.as_wss(Addressable::URI.escape(url))
       return {} unless wss_url.present?
 
       parsed_wss_url = URI.parse(wss_url)
@@ -33,7 +33,7 @@ module Ci
 
       port = port.presence || DEFAULT_PORT_NAME
       service = service.presence || DEFAULT_SERVICE_NAME
-      parsed_url = URI.parse(Addressable::URI.escape(self.url))
+      parsed_url = URI.parse(Addressable::URI.escape(url))
       parsed_url.path += "/proxy/#{service}/#{port}/#{path}"
       subprotocols = subprotocols.presence || ::Ci::BuildRunnerSession::TERMINAL_SUBPROTOCOL
 
diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb
index c5f6e54c336..541a8b5bffa 100644
--- a/app/models/ci/build_trace_chunk.rb
+++ b/app/models/ci/build_trace_chunk.rb
@@ -108,7 +108,7 @@ module Ci
       raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
       return if offset == size # Skip the following process as it doesn't affect anything
 
-      self.append(+"", offset)
+      append(+"", offset)
     end
 
     def append(new_data, offset)
@@ -166,7 +166,7 @@ module Ci
         raise FailedToPersistDataError, 'Modifed build trace chunk detected' if has_changes_to_save?
 
         self.class.with_read_consistency(build) do
-          self.reset.then(&:unsafe_persist_data!)
+          reset.then(&:unsafe_persist_data!)
         end
       end
     rescue FailedToObtainLockError
@@ -205,9 +205,9 @@ module Ci
     end
 
     def <=>(other)
-      return unless self.build_id == other.build_id
+      return unless build_id == other.build_id
 
-      self.chunk_index <=> other.chunk_index
+      chunk_index <=> other.chunk_index
     end
 
     protected
diff --git a/app/models/ci/deleted_object.rb b/app/models/ci/deleted_object.rb
index d36646aba66..2b5452c803a 100644
--- a/app/models/ci/deleted_object.rb
+++ b/app/models/ci/deleted_object.rb
@@ -21,7 +21,7 @@ module Ci
         accumulator << record if record[:store_dir] && record[:file]
       end
 
-      self.insert_all(attributes) if attributes.any?
+      insert_all(attributes) if attributes.any?
     end
 
     def delete_file_from_storage
diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb
index 508aaa5a63c..b03c46a164f 100644
--- a/app/models/ci/group_variable.rb
+++ b/app/models/ci/group_variable.rb
@@ -3,9 +3,11 @@
 module Ci
   class GroupVariable < Ci::ApplicationRecord
     include Ci::HasVariable
-    include Presentable
     include Ci::Maskable
     include Ci::RawVariable
+    include Limitable
+    include Presentable
+
     prepend HasEnvironmentScope
 
     belongs_to :group, class_name: "::Group"
@@ -21,6 +23,9 @@ module Ci
     scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) }
     scope :for_groups, ->(group_ids) { where(group_id: group_ids) }
 
+    self.limit_name = 'group_ci_variables'
+    self.limit_scope = :group
+
     def audit_details
       key
     end
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 0dca5b18a24..89a3d269a43 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -134,15 +134,17 @@ module Ci
     belongs_to :project
     belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
 
-    # We will start using this column once we complete https://gitlab.com/gitlab-org/gitlab/-/issues/285597
-    ignore_column :original_filename, remove_with: '14.7', remove_after: '2022-11-22'
-
     mount_file_store_uploader JobArtifactUploader, skip_store_file: true
 
     before_save :set_size, if: :file_changed?
     after_save :store_file_in_transaction!, unless: :store_after_commit?
+
+    after_create_commit :log_create
+
     after_commit :store_file_after_transaction!, on: [:create, :update], if: :store_after_commit?
 
+    after_destroy_commit :log_destroy
+
     validates :job, presence: true
     validates :file_format, presence: true, unless: :trace?, on: :create
     validate :validate_file_format!, unless: :trace?, on: :create
@@ -384,6 +386,14 @@ module Ci
       # Use job.project to avoid extra DB query for project
       job.project.pending_delete?
     end
+
+    def log_create
+      Gitlab::Ci::Artifacts::Logger.log_created(self)
+    end
+
+    def log_destroy
+      Gitlab::Ci::Artifacts::Logger.log_deleted(self, __method__)
+    end
   end
 end
 
diff --git a/app/models/ci/job_token/allowlist.rb b/app/models/ci/job_token/allowlist.rb
index 9e9a0a68ebd..618dc2da05c 100644
--- a/app/models/ci/job_token/allowlist.rb
+++ b/app/models/ci/job_token/allowlist.rb
@@ -17,6 +17,15 @@ module Ci
         Project.from_union(target_projects, remove_duplicates: false)
       end
 
+      def add!(target_project, user:)
+        Ci::JobToken::ProjectScopeLink.create!(
+          source_project: @source_project,
+          direction: @direction,
+          target_project: target_project,
+          added_by: user
+        )
+      end
+
       private
 
       def source_links
diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb
index b784f93651a..96e370bba1e 100644
--- a/app/models/ci/job_token/project_scope_link.rb
+++ b/app/models/ci/job_token/project_scope_link.rb
@@ -1,24 +1,31 @@
 # frozen_string_literal: true
 
-# The connection between a source project (which defines the job token scope)
-# and a target project which is the one allowed to be accessed by the job token.
+# The connection between a source project (which the job token scope's allowlist applies too)
+# and a target project which is added to the scope's allowlist.
 
 module Ci
   module JobToken
     class ProjectScopeLink < Ci::ApplicationRecord
       self.table_name = 'ci_job_token_project_scope_links'
 
+      PROJECT_LINK_DIRECTIONAL_LIMIT = 100
+
       belongs_to :source_project, class_name: 'Project'
+      # the project added to the scope's allowlist
       belongs_to :target_project, class_name: 'Project'
       belongs_to :added_by, class_name: 'User'
 
-      scope :with_source, ->(project) { where(source_project: project) }
-      scope :with_target, ->(project) { where(target_project: project) }
+      scope :with_access_direction, ->(direction) { where(direction: direction) }
+      scope :with_source, ->(project)   { where(source_project: project) }
+      scope :with_target, ->(project)   { where(target_project: project) }
 
       validates :source_project, presence: true
       validates :target_project, presence: true
       validate :not_self_referential_link
+      validate :source_project_under_link_limit, on: :create
 
+      # When outbound the target project is allowed to be accessed by the source job token.
+      # When inbound the source project is allowed to be accessed by the target job token.
       enum direction: {
         outbound: 0,
         inbound: 1
@@ -37,6 +44,16 @@ module Ci
           self.errors.add(:target_project, _("can't be the same as the source project"))
         end
       end
+
+      def source_project_under_link_limit
+        return unless source_project
+
+        existing_links_count = self.class.with_source(source_project).with_access_direction(direction).count
+
+        if existing_links_count >= PROJECT_LINK_DIRECTIONAL_LIMIT
+          errors.add(:source_project, "exceeds the allowable number of project links in this direction")
+        end
+      end
     end
   end
 end
diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb
index e320c0f92d1..20775077bd8 100644
--- a/app/models/ci/job_token/scope.rb
+++ b/app/models/ci/job_token/scope.rb
@@ -2,18 +2,17 @@
 
 # This model represents the scope of access for a CI_JOB_TOKEN.
 #
-# A scope is initialized with a project.
+# A scope is initialized with a current project.
 #
 # Projects can be added to the scope by adding ScopeLinks to
 # create an allowlist of projects in either access direction (inbound, outbound).
 #
-# Currently, projects in the outbound allowlist can be accessed via the token
-# in the source project.
+# Projects in the outbound allowlist can be accessed via the current project's job token.
 #
-# TODO(Issue #346298) Projects in the inbound allowlist can use their token to access
-# the source project.
+# Projects in the inbound allowlist can use their project's job token to
+# access the current project.
 #
-# CI_JOB_TOKEN should be considered untrusted without these features enabled.
+# CI_JOB_TOKEN should be considered untrusted without a scope enabled.
 #
 
 module Ci
@@ -25,34 +24,70 @@ module Ci
         @current_project = current_project
       end
 
-      def allows?(accessed_project)
-        self_referential?(accessed_project) || outbound_allows?(accessed_project)
+      def accessible?(accessed_project)
+        self_referential?(accessed_project) || (
+          outbound_accessible?(accessed_project) &&
+          inbound_accessible?(accessed_project)
+        )
       end
 
       def outbound_projects
         outbound_allowlist.projects
       end
 
-      # Deprecated: use outbound_projects, TODO(Issue #346298) remove references to all_project
-      def all_projects
-        outbound_projects
+      def inbound_projects
+        inbound_allowlist.projects
+      end
+
+      def add!(added_project, user:, direction:)
+        case direction
+        when :inbound
+          inbound_allowlist.add!(added_project, user: user)
+        when :outbound
+          outbound_allowlist.add!(added_project, user: user)
+        end
       end
 
       private
 
-      def outbound_allows?(accessed_project)
+      def outbound_accessible?(accessed_project)
         # if the setting is disabled any project is considered to be in scope.
-        return true unless @current_project.ci_outbound_job_token_scope_enabled?
+        return true unless current_project.ci_outbound_job_token_scope_enabled?
 
         outbound_allowlist.includes?(accessed_project)
       end
 
+      def inbound_accessible?(accessed_project)
+        # if the flag or setting is disabled any project is considered to be in scope.
+        return true unless Feature.enabled?(:ci_inbound_job_token_scope, accessed_project)
+        return true unless accessed_project.ci_inbound_job_token_scope_enabled?
+
+        inbound_linked_as_accessible?(accessed_project)
+      end
+
+      # We don't check the inbound allowlist here. That is because
+      # the access check starts from the current project but the inbound
+      # allowlist contains projects that can access the current project.
+      def inbound_linked_as_accessible?(accessed_project)
+        inbound_accessible_projects(accessed_project).includes?(current_project)
+      end
+
+      def inbound_accessible_projects(accessed_project)
+        Ci::JobToken::Allowlist.new(accessed_project, direction: :inbound)
+      end
+
+      # User created list of projects allowed to access the current project
+      def inbound_allowlist
+        Ci::JobToken::Allowlist.new(current_project, direction: :inbound)
+      end
+
+      # User created list of projects that can be accessed from the current project
       def outbound_allowlist
-        Ci::JobToken::Allowlist.new(@current_project, direction: :outbound)
+        Ci::JobToken::Allowlist.new(current_project, direction: :outbound)
       end
 
       def self_referential?(accessed_project)
-        @current_project.id == accessed_project.id
+        current_project.id == accessed_project.id
       end
     end
   end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index eab2ab69e44..bd426e02b9c 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -355,7 +355,7 @@ module Ci
     scope :for_name, -> (name) do
       name_column = Ci::PipelineMetadata.arel_table[:name]
 
-      joins(:pipeline_metadata).where(name_column.lower.eq(name.downcase))
+      joins(:pipeline_metadata).where(name_column.eq(name))
     end
     scope :created_after, -> (time) { where(arel_table[:created_at].gt(time)) }
     scope :created_before_id, -> (id) { where(arel_table[:id].lt(id)) }
@@ -498,6 +498,10 @@ module Ci
       100
     end
 
+    def self.object_hierarchy(relation, options = {})
+      ::Gitlab::Ci::PipelineObjectHierarchy.new(relation, options: options)
+    end
+
     def uses_needs?
       processables.where(scheduling_type: :dag).any?
     end
@@ -841,97 +845,6 @@ module Ci
       end
     end
 
-    def predefined_variables
-      Gitlab::Ci::Variables::Collection.new.tap do |variables|
-        variables.append(key: 'CI_PIPELINE_IID', value: iid.to_s)
-        variables.append(key: 'CI_PIPELINE_SOURCE', value: source.to_s)
-        variables.append(key: 'CI_PIPELINE_CREATED_AT', value: created_at&.iso8601)
-
-        variables.concat(predefined_commit_variables)
-        variables.concat(predefined_merge_request_variables)
-
-        if open_merge_requests_refs.any?
-          variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: open_merge_requests_refs.join(','))
-        end
-
-        variables.append(key: 'CI_GITLAB_FIPS_MODE', value: 'true') if Gitlab::FIPS.enabled?
-
-        variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if has_kubernetes_active?
-        variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if freeze_period?
-
-        if external_pull_request_event? && external_pull_request
-          variables.concat(external_pull_request.predefined_variables)
-        end
-      end
-    end
-
-    def predefined_commit_variables
-      strong_memoize(:predefined_commit_variables) do
-        Gitlab::Ci::Variables::Collection.new.tap do |variables|
-          next variables unless sha.present?
-
-          variables.append(key: 'CI_COMMIT_SHA', value: sha)
-          variables.append(key: 'CI_COMMIT_SHORT_SHA', value: short_sha)
-          variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: before_sha)
-          variables.append(key: 'CI_COMMIT_REF_NAME', value: source_ref)
-          variables.append(key: 'CI_COMMIT_REF_SLUG', value: source_ref_slug)
-          variables.append(key: 'CI_COMMIT_BRANCH', value: ref) if branch?
-          variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s)
-          variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s)
-          variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s)
-          variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!protected_ref?).to_s)
-          variables.append(key: 'CI_COMMIT_TIMESTAMP', value: git_commit_timestamp.to_s)
-          variables.append(key: 'CI_COMMIT_AUTHOR', value: git_author_full_text.to_s)
-
-          # legacy variables
-          variables.append(key: 'CI_BUILD_REF', value: sha)
-          variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha)
-          variables.append(key: 'CI_BUILD_REF_NAME', value: source_ref)
-          variables.append(key: 'CI_BUILD_REF_SLUG', value: source_ref_slug)
-
-          variables.concat(predefined_commit_tag_variables)
-        end
-      end
-    end
-
-    def predefined_merge_request_variables
-      strong_memoize(:predefined_merge_request_variables) do
-        Gitlab::Ci::Variables::Collection.new.tap do |variables|
-          next variables unless merge_request?
-
-          variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: merge_request_event_type.to_s)
-          variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', value: source_sha.to_s)
-          variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: target_sha.to_s)
-
-          diff = self.merge_request_diff
-          if diff.present?
-            variables.append(key: 'CI_MERGE_REQUEST_DIFF_ID', value: diff.id.to_s)
-            variables.append(key: 'CI_MERGE_REQUEST_DIFF_BASE_SHA', value: diff.base_commit_sha)
-          end
-
-          variables.concat(merge_request.predefined_variables)
-        end
-      end
-    end
-
-    def predefined_commit_tag_variables
-      strong_memoize(:predefined_commit_ref_variables) do
-        Gitlab::Ci::Variables::Collection.new.tap do |variables|
-          next variables unless tag?
-
-          git_tag = project.repository.find_tag(ref)
-
-          next variables unless git_tag
-
-          variables.append(key: 'CI_COMMIT_TAG', value: ref)
-          variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message)
-
-          # legacy variable
-          variables.append(key: 'CI_BUILD_TAG', value: ref)
-        end
-      end
-    end
-
     def queued_duration
       return unless started_at
 
@@ -1403,6 +1316,12 @@ module Ci
       (Time.current - created_at).ceil / 60
     end
 
+    def merge_request_diff
+      return unless merge_request?
+
+      merge_request.merge_request_diff_for(merge_request_diff_sha)
+    end
+
     private
 
     def cancel_jobs(jobs, retries: 1, auto_canceled_by_pipeline_id: nil)
@@ -1455,12 +1374,6 @@ module Ci
       end
     end
 
-    def merge_request_diff
-      return unless merge_request?
-
-      merge_request.merge_request_diff_for(merge_request_diff_sha)
-    end
-
     def push_details
       strong_memoize(:push_details) do
         Gitlab::Git::Push.new(project, before_sha, sha, git_ref)
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index bac85b6095e..09ac0fa69e7 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -15,6 +15,8 @@ module Ci
     include EachBatch
     include Ci::HasRunnerExecutor
 
+    extend ::Gitlab::Utils::Override
+
     add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration
 
     enum access_level: {
@@ -28,6 +30,14 @@ module Ci
       project_type: 3
     }
 
+    enum registration_type: {
+      registration_token: 0,
+      authenticated_user: 1
+    }, _suffix: true
+
+    # Prefix assigned to runners created from the UI, instead of registered via the command line
+    CREATED_RUNNER_TOKEN_PREFIX = 'glrt-'
+
     # This `ONLINE_CONTACT_TIMEOUT` needs to be larger than
     #   `RUNNER_QUEUE_EXPIRY_TIME+UPDATE_CONTACT_COLUMN_EVERY`
     #
@@ -179,6 +189,7 @@ module Ci
     validate :tag_constraints
     validates :access_level, presence: true
     validates :runner_type, presence: true
+    validates :registration_type, presence: true
 
     validate :no_projects, unless: :project_type?
     validate :no_groups, unless: :group_type?
@@ -373,7 +384,10 @@ module Ci
     end
 
     def short_sha
-      token[0...8] if token
+      return unless token
+
+      start_index = authenticated_user_registration_type? ? CREATED_RUNNER_TOKEN_PREFIX.length : 0
+      token[start_index..start_index + 8]
     end
 
     def tag_list
@@ -474,6 +488,17 @@ module Ci
       end
     end
 
+    override :format_token
+    def format_token(token)
+      return token if registration_token_registration_type?
+
+      "#{CREATED_RUNNER_TOKEN_PREFIX}#{token}"
+    end
+
+    def ensure_machine(system_xid, &blk)
+      RunnerMachine.safe_find_or_create_by!(runner_id: id, system_xid: system_xid.to_s, &blk) # rubocop: disable Performance/ActiveRecordSubtransactionMethods
+    end
+
     private
 
     scope :with_upgrade_status, ->(upgrade_status) do
@@ -566,6 +591,9 @@ module Ci
       end
     end
 
+    # TODO Remove in 16.0 when runners are known to send a system_id
+    # For now, heartbeats with version updates might result in two Sidekiq jobs being queued if a runner has a system_id
+    # This is not a problem since the jobs are deduplicated on the version
     def schedule_runner_version_update
       return unless version
 
diff --git a/app/models/ci/runner_machine.rb b/app/models/ci/runner_machine.rb
index 1dd997a8ee1..e52659a011f 100644
--- a/app/models/ci/runner_machine.rb
+++ b/app/models/ci/runner_machine.rb
@@ -3,12 +3,24 @@
 module Ci
   class RunnerMachine < Ci::ApplicationRecord
     include FromUnion
+    include RedisCacheable
     include Ci::HasRunnerExecutor
+    include IgnorableColumns
+
+    ignore_column :machine_xid, remove_with: '15.11', remove_after: '2022-03-22'
+
+    # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner Machine DB entry can be updated
+    UPDATE_CONTACT_COLUMN_EVERY = 40.minutes..55.minutes
 
     belongs_to :runner
 
+    has_many :build_metadata, class_name: 'Ci::BuildMetadata'
+    has_many :builds, through: :build_metadata, class_name: 'Ci::Build'
+    belongs_to :runner_version, inverse_of: :runner_machines, primary_key: :version, foreign_key: :version,
+               class_name: 'Ci::RunnerVersion'
+
     validates :runner, presence: true
-    validates :machine_xid, presence: true, length: { maximum: 64 }
+    validates :system_xid, presence: true, length: { maximum: 64 }
     validates :version, length: { maximum: 2048 }
     validates :revision, length: { maximum: 255 }
     validates :platform, length: { maximum: 255 }
@@ -16,6 +28,8 @@ module Ci
     validates :ip_address, length: { maximum: 1024 }
     validates :config, json_schema: { filename: 'ci_runner_config' }
 
+    cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type
+
     # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner machine
     # will be considered stale
     STALE_TIMEOUT = 7.days
@@ -29,5 +43,46 @@ module Ci
         where(contacted_some_time_ago),
         remove_duplicates: false).where(created_some_time_ago)
     end
+
+    def heartbeat(values)
+      ##
+      # We can safely ignore writes performed by a runner heartbeat. We do
+      # not want to upgrade database connection proxy to use the primary
+      # database after heartbeat write happens.
+      #
+      ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do
+        values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {}
+        values[:contacted_at] = Time.current
+        if values.include?(:executor)
+          values[:executor_type] = Ci::Runner::EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown)
+        end
+
+        version_changed = values.include?(:version) && values[:version] != version
+
+        cache_attributes(values)
+
+        schedule_runner_version_update if version_changed
+
+        # We save data without validation, it will always change due to `contacted_at`
+        update_columns(values) if persist_cached_data?
+      end
+    end
+
+    private
+
+    def persist_cached_data?
+      # Use a random threshold to prevent beating DB updates.
+      contacted_at_max_age = Random.rand(UPDATE_CONTACT_COLUMN_EVERY)
+
+      real_contacted_at = read_attribute(:contacted_at)
+      real_contacted_at.nil? ||
+        (Time.current - real_contacted_at) >= contacted_at_max_age
+    end
+
+    def schedule_runner_version_update
+      return unless version
+
+      Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version)
+    end
   end
 end
diff --git a/app/models/ci/runner_version.rb b/app/models/ci/runner_version.rb
index bbde98ee591..ec42f46b165 100644
--- a/app/models/ci/runner_version.rb
+++ b/app/models/ci/runner_version.rb
@@ -8,24 +8,23 @@ module Ci
     enum_with_nil status: {
       not_processed: nil,
       invalid_version: -1,
-      not_available: 1,
+      unavailable: 1,
       available: 2,
       recommended: 3
     }
 
     STATUS_DESCRIPTIONS = {
       invalid_version: 'Runner version is not valid.',
-      not_available: 'Upgrade is not available for the runner.',
+      unavailable: 'Upgrade is not available for the runner.',
       available: 'Upgrade is available for the runner.',
       recommended: 'Upgrade is available and recommended for the runner.'
     }.freeze
 
-    # Override auto generated negative scope (from available) so the scope has expected behavior
-    scope :not_available, -> { where(status: :not_available) }
+    has_many :runner_machines, inverse_of: :runner_version, foreign_key: :version, class_name: 'Ci::RunnerMachine'
 
     # This scope returns all versions that might need recalculating. For instance, once a version is considered
     # :recommended, it normally doesn't change status even if the instance is upgraded
-    scope :potentially_outdated, -> { where(status: [nil, :not_available, :available]) }
+    scope :potentially_outdated, -> { where(status: [nil, :unavailable, :available]) }
 
     validates :version, length: { maximum: 2048 }
   end
diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb
index 1e6c48bbef5..5e273e0fd4b 100644
--- a/app/models/ci/secure_file.rb
+++ b/app/models/ci/secure_file.rb
@@ -35,7 +35,7 @@ module Ci
     end
 
     def file_extension
-      File.extname(name).delete_prefix('.')
+      File.extname(name).delete_prefix('.').presence
     end
 
     def metadata_parsable?
diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb
index 1092b9c9564..1b2a7dc3fe4 100644
--- a/app/models/ci/trigger.rb
+++ b/app/models/ci/trigger.rb
@@ -21,8 +21,18 @@ module Ci
     validates :token, presence: true, uniqueness: true
     validates :owner, presence: true
 
+    attr_encrypted :encrypted_token_tmp,
+      attribute: :encrypted_token,
+      mode: :per_attribute_iv,
+      algorithm: 'aes-256-gcm',
+      key: Settings.attr_encrypted_db_key_base_32,
+      encode: false,
+      encode_vi: false
+
     before_validation :set_default_values
 
+    before_save :copy_token_to_encrypted_token
+
     def set_default_values
       self.token = "#{TRIGGER_TOKEN_PREFIX}#{SecureRandom.hex(20)}" if self.token.blank?
     end
@@ -42,6 +52,12 @@ module Ci
     def can_access_project?
       Ability.allowed?(self.owner, :create_build, project)
     end
+
+    private
+
+    def copy_token_to_encrypted_token
+      self.encrypted_token_tmp = token
+    end
   end
 end
 
diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb
index f4e17b5d812..23fe89c38df 100644
--- a/app/models/ci/variable.rb
+++ b/app/models/ci/variable.rb
@@ -3,9 +3,11 @@
 module Ci
   class Variable < Ci::ApplicationRecord
     include Ci::HasVariable
-    include Presentable
     include Ci::Maskable
     include Ci::RawVariable
+    include Limitable
+    include Presentable
+
     prepend HasEnvironmentScope
 
     belongs_to :project
@@ -20,6 +22,9 @@ module Ci
     scope :unprotected, -> { where(protected: false) }
     scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) }
 
+    self.limit_name = 'project_ci_variables'
+    self.limit_scope = :project
+
     def audit_details
       key
     end