diff options
Diffstat (limited to 'app/models/ci')
| -rw-r--r-- | app/models/ci/build.rb | 27 | ||||
| -rw-r--r-- | app/models/ci/job_artifact.rb | 9 | ||||
| -rw-r--r-- | app/models/ci/namespace_mirror.rb | 37 | ||||
| -rw-r--r-- | app/models/ci/pending_build.rb | 12 | ||||
| -rw-r--r-- | app/models/ci/pipeline.rb | 62 | ||||
| -rw-r--r-- | app/models/ci/project_mirror.rb | 16 | ||||
| -rw-r--r-- | app/models/ci/runner.rb | 77 | ||||
| -rw-r--r-- | app/models/ci/runner_namespace.rb | 1 | ||||
| -rw-r--r-- | app/models/ci/runner_project.rb | 1 | ||||
| -rw-r--r-- | app/models/ci/stage.rb | 1 |
10 files changed, 179 insertions, 64 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 3fdc44bccf3..428e440afba 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -10,6 +10,7 @@ module Ci include Presentable include Importable include Ci::HasRef + extend ::Gitlab::Utils::Override BuildArchivedError = Class.new(StandardError) @@ -58,7 +59,7 @@ module Ci has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build - has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', dependent: :nullify, inverse_of: :build, foreign_key: :ci_build_id # rubocop:disable Cop/ActiveRecordDependent + has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', inverse_of: :build, foreign_key: :ci_build_id accepts_nested_attributes_for :runner_session, update_only: true accepts_nested_attributes_for :job_variables @@ -164,6 +165,7 @@ module Ci scope :with_artifacts_not_expired, -> { with_downloadable_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.current) } scope :with_expired_artifacts, -> { with_downloadable_artifacts.where('artifacts_expire_at < ?', Time.current) } + scope :with_pipeline_locked_artifacts, -> { joins(:pipeline).where('pipeline.locked': Ci::Pipeline.lockeds[:artifacts_locked]) } scope :last_month, -> { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, -> { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } scope :scheduled_actions, -> { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) } @@ -188,8 +190,6 @@ module Ci scope :without_coverage, -> { where(coverage: nil) } scope :with_coverage_regex, -> { where.not(coverage_regex: nil) } - scope :for_project, -> (project_id) { where(project_id: project_id) } - acts_as_taggable add_authentication_token_field :token, encrypted: :required @@ -286,6 +286,7 @@ module Ci build.run_after_commit do BuildQueueWorker.perform_async(id) + BuildHooksWorker.perform_async(id) end end @@ -451,7 +452,7 @@ module Ci end def retryable? - return false if retried? || archived? + return false if retried? || archived? || deployment_rejected? success? || failed? || canceled? end @@ -722,6 +723,14 @@ module Ci self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token) end + # acts_as_taggable uses this method create/remove tags with contexts + # defined by taggings and to get those contexts it executes a query. + # We don't use any other contexts except `tags`, so we don't need it. + override :custom_contexts + def custom_contexts + [] + end + def tag_list if tags.loaded? tags.map(&:name) @@ -1074,6 +1083,16 @@ module Ci runner&.instance_type? end + def job_variables_attributes + strong_memoize(:job_variables_attributes) do + job_variables.internal_source.map do |variable| + variable.attributes.except('id', 'job_id', 'encrypted_value', 'encrypted_value_iv').tap do |attrs| + attrs[:value] = variable.value + end + end + end + end + protected def run_status_commit_hooks! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index ec1137920ef..e6dd62fab34 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -2,6 +2,7 @@ module Ci class JobArtifact < Ci::ApplicationRecord + include IgnorableColumns include AfterCommitQueue include ObjectStorage::BackgroundMove include UpdateProjectStatistics @@ -120,6 +121,9 @@ module Ci belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + # We will start using this column once we complete https://gitlab.com/gitlab-org/gitlab/-/issues/285597 + ignore_column :original_filename, remove_with: '14.7', remove_after: '2022-11-22' + mount_file_store_uploader JobArtifactUploader skip_callback :save, :after, :store_file!, if: :store_after_commit? @@ -133,6 +137,7 @@ module Ci scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } + scope :for_job_ids, ->(job_ids) { where(job_id: job_ids) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } scope :with_job, -> { joins(:job).includes(:job) } @@ -266,6 +271,10 @@ module Ci self.where(project: project).sum(:size) end + def self.distinct_job_ids + distinct.pluck(:job_id) + end + ## # FastDestroyAll concerns # rubocop: disable CodeReuse/ServiceClass diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb new file mode 100644 index 00000000000..8a4be3139e8 --- /dev/null +++ b/app/models/ci/namespace_mirror.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Ci + # This model represents a record in a shadow table of the main database's namespaces table. + # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. + class NamespaceMirror < ApplicationRecord + belongs_to :namespace + + scope :contains_namespace, -> (id) do + where('traversal_ids @> ARRAY[?]::int[]', id) + end + + class << self + def sync!(event) + namespace = event.namespace + traversal_ids = namespace.self_and_ancestor_ids(hierarchy_order: :desc) + + upsert({ namespace_id: event.namespace_id, traversal_ids: traversal_ids }, + unique_by: :namespace_id) + + # It won't be necessary once we remove `sync_traversal_ids`. + # More info: https://gitlab.com/gitlab-org/gitlab/-/issues/347541 + sync_children_namespaces!(event.namespace_id, traversal_ids) + end + + private + + def sync_children_namespaces!(namespace_id, traversal_ids) + contains_namespace(namespace_id) + .where.not(namespace_id: namespace_id) + .update_all( + "traversal_ids = ARRAY[#{sanitize_sql(traversal_ids.join(','))}]::int[] || traversal_ids[array_position(traversal_ids, #{sanitize_sql(namespace_id)}) + 1:]" + ) + end + end + end +end diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index ccad6290fac..41dc74ef050 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -30,6 +30,10 @@ module Ci self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) end + def maintain_denormalized_data? + ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + end + private def args_from_build(build) @@ -42,15 +46,9 @@ module Ci namespace: project.namespace } - if Feature.enabled?(:ci_pending_builds_maintain_tags_data, type: :development, default_enabled: :yaml) + if maintain_denormalized_data? args.store(:tag_ids, build.tags_ids) - end - - if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) args.store(:instance_runners_enabled, shared_runners_enabled?(project)) - end - - if Feature.enabled?(:ci_pending_builds_maintain_namespace_traversal_ids, type: :development, default_enabled: :yaml) args.store(:namespace_traversal_ids, project.namespace.traversal_ids) if group_runners_enabled?(project) end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index a29aa756e38..a90bd739741 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -63,6 +63,7 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses_ordered_by_stage, -> { latest.order(:stage_idx, :stage) }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses, -> { latest }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline + has_many :statuses_order_id_desc, -> { order_id_desc }, class_name: 'CommitStatus', foreign_key: :commit_id has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline @@ -82,8 +83,6 @@ module Ci # Merge requests for which the current pipeline is running against # the merge request's latest commit. has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest' - has_many :package_build_infos, class_name: 'Packages::BuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent - has_many :package_file_build_infos, class_name: 'Packages::PackageFileBuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline @@ -236,7 +235,18 @@ module Ci pipeline.run_after_commit do PipelineHooksWorker.perform_async(pipeline.id) - ExpirePipelineCacheWorker.perform_async(pipeline.id) + + if pipeline.project.jira_subscription_exists? + # Passing the seq-id ensures this is idempotent + seq_id = ::Atlassian::JiraConnect::Client.generate_update_sequence_id + ::JiraConnect::SyncBuildsWorker.perform_async(pipeline.id, seq_id) + end + + if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, pipeline.project, default_enabled: :yaml) + Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass + else + ExpirePipelineCacheWorker.perform_async(pipeline.id) + end end end @@ -271,14 +281,6 @@ module Ci end end - after_transition any => any do |pipeline| - pipeline.run_after_commit do - # Passing the seq-id ensures this is idempotent - seq_id = ::Atlassian::JiraConnect::Client.generate_update_sequence_id - ::JiraConnect::SyncBuildsWorker.perform_async(pipeline.id, seq_id) - end - end - after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline| pipeline.run_after_commit do ::Ci::TestFailureHistoryService.new(pipeline).async.perform_if_needed # rubocop: disable CodeReuse/ServiceClass @@ -643,7 +645,7 @@ module Ci def coverage coverage_array = latest_statuses.map(&:coverage).compact if coverage_array.size >= 1 - '%.2f' % (coverage_array.reduce(:+) / coverage_array.size) + coverage_array.reduce(:+) / coverage_array.size end end @@ -947,22 +949,16 @@ module Ci end def environments_in_self_and_descendants - if ::Feature.enabled?(:avoid_cross_joins_environments_in_self_and_descendants, default_enabled: :yaml) - # We limit to 100 unique environments for application safety. - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 - expanded_environment_names = - builds_in_self_and_descendants.joins(:metadata) - .where.not('ci_builds_metadata.expanded_environment_name' => nil) - .distinct('ci_builds_metadata.expanded_environment_name') - .limit(100) - .pluck(:expanded_environment_name) - - Environment.where(project: project, name: expanded_environment_names) - else - environment_ids = self_and_descendants.joins(:deployments).select(:'deployments.environment_id') + # We limit to 100 unique environments for application safety. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 + expanded_environment_names = + builds_in_self_and_descendants.joins(:metadata) + .where.not('ci_builds_metadata.expanded_environment_name' => nil) + .distinct('ci_builds_metadata.expanded_environment_name') + .limit(100) + .pluck(:expanded_environment_name) - Environment.where(id: environment_ids) - end + Environment.where(project: project, name: expanded_environment_names).with_deployment(sha) end # With multi-project and parent-child pipelines @@ -1276,18 +1272,18 @@ module Ci self.builds.latest.build_matchers(project) end - def predefined_vars_in_builder_enabled? - strong_memoize(:predefined_vars_in_builder_enabled) do - Feature.enabled?(:ci_predefined_vars_in_builder, project, default_enabled: :yaml) - end - end - def authorized_cluster_agents strong_memoize(:authorized_cluster_agents) do ::Clusters::AgentAuthorizationsFinder.new(project).execute.map(&:agent) end end + def create_deployment_in_separate_transaction? + strong_memoize(:create_deployment_in_separate_transaction) do + ::Feature.enabled?(:create_deployment_in_separate_transaction, project, default_enabled: :yaml) + end + end + private def add_message(severity, content) diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb new file mode 100644 index 00000000000..d6aaa3f50c1 --- /dev/null +++ b/app/models/ci/project_mirror.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Ci + # This model represents a shadow table of the main database's projects table. + # It allows us to navigate the project and namespace hierarchy on the ci database. + class ProjectMirror < ApplicationRecord + belongs_to :project + + class << self + def sync!(event) + upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id }, + unique_by: :project_id) + end + end + end +end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8a3025e5608..a80fd02080f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -12,7 +12,6 @@ module Ci include Gitlab::Utils::StrongMemoize include TaggableQueries include Presentable - include LooseForeignKey add_authentication_token_field :token, encrypted: :optional @@ -27,6 +26,21 @@ module Ci project_type: 3 } + enum executor_type: { + unknown: 0, + custom: 1, + shell: 2, + docker: 3, + docker_windows: 4, + docker_ssh: 5, + ssh: 6, + parallels: 7, + virtualbox: 8, + docker_machine: 9, + docker_ssh_machine: 10, + kubernetes: 11 + }, _suffix: true + # This `ONLINE_CONTACT_TIMEOUT` needs to be larger than # `RUNNER_QUEUE_EXPIRY_TIME+UPDATE_CONTACT_COLUMN_EVERY` # @@ -40,9 +54,12 @@ module Ci # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner DB entry can be updated UPDATE_CONTACT_COLUMN_EVERY = (40.minutes..55.minutes).freeze + # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner will be considered stale + STALE_TIMEOUT = 3.months + AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze AVAILABLE_TYPES = runner_types.keys.freeze - AVAILABLE_STATUSES = %w[active paused online offline not_connected].freeze + AVAILABLE_STATUSES = %w[active paused online offline not_connected never_contacted stale].freeze # TODO: Remove in %15.0: active, paused, not_connected. Relevant issues: https://gitlab.com/gitlab-org/gitlab/-/issues/347303, https://gitlab.com/gitlab-org/gitlab/-/issues/347305, https://gitlab.com/gitlab-org/gitlab/-/issues/344648 AVAILABLE_SCOPES = (AVAILABLE_TYPES_LEGACY + AVAILABLE_TYPES + AVAILABLE_STATUSES).freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze @@ -58,12 +75,14 @@ module Ci before_save :ensure_token - scope :active, -> { where(active: true) } - scope :paused, -> { where(active: false) } + scope :active, -> (value = true) { where(active: value) } + scope :paused, -> { active(false) } scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) } - scope :recent, -> { where('ci_runners.created_at > :date OR ci_runners.contacted_at > :date', date: 3.months.ago) } + scope :recent, -> { where('ci_runners.created_at >= :date OR ci_runners.contacted_at >= :date', date: stale_deadline) } + scope :stale, -> { where('ci_runners.created_at < :date AND (ci_runners.contacted_at IS NULL OR ci_runners.contacted_at < :date)', date: stale_deadline) } scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } - scope :not_connected, -> { where(contacted_at: nil) } + scope :not_connected, -> { where(contacted_at: nil) } # TODO: Remove in 15.0 + scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } scope :with_recent_runner_queue, -> { where('contacted_at > ?', recent_queue_deadline) } @@ -78,10 +97,7 @@ module Ci scope :belonging_to_group, -> (group_id, include_ancestors: false) { groups = ::Group.where(id: group_id) - - if include_ancestors - groups = Gitlab::ObjectHierarchy.new(groups).base_and_ancestors - end + groups = groups.self_and_ancestors if include_ancestors joins(:runner_namespaces) .where(ci_runner_namespaces: { namespace_id: groups }) @@ -102,10 +118,9 @@ module Ci scope :belonging_to_parent_group_of_project, -> (project_id) { project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) - hierarchy_groups = Gitlab::ObjectHierarchy.new(project_groups).base_and_ancestors joins(:groups) - .where(namespaces: { id: hierarchy_groups }) + .where(namespaces: { id: project_groups.self_and_ancestors.as_ids }) .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') } @@ -152,7 +167,7 @@ module Ci after_destroy :cleanup_runner_queue - cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at + cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout, error_message: 'Maximum job timeout has a value which could not be accepted' @@ -168,8 +183,6 @@ module Ci validates :config, json_schema: { filename: 'ci_runner_config' } - loose_foreign_key :clusters_applications_runners, :runner_id, on_delete: :async_nullify - # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens. @@ -185,6 +198,10 @@ module Ci ONLINE_CONTACT_TIMEOUT.ago end + def self.stale_deadline + STALE_TIMEOUT.ago + end + def self.recent_queue_deadline # we add queue expiry + online # - contacted_at can be updated at any time within this interval @@ -273,8 +290,17 @@ module Ci contacted_at && contacted_at > self.class.online_contact_time_deadline end - def status - return :not_connected unless contacted_at + def stale? + return false unless created_at + + [created_at, contacted_at].compact.max < self.class.stale_deadline + end + + def status(legacy_mode = nil) + return deprecated_rest_status if legacy_mode == '14.5' + + return :stale if stale? + return :never_contacted unless contacted_at online? ? :online : :offline end @@ -387,8 +413,9 @@ module Ci # database after heartbeat write happens. # ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do - values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config) || {} + values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} values[:contacted_at] = Time.current + values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) cache_attributes(values) @@ -413,6 +440,20 @@ module Ci private + EXECUTOR_NAME_TO_TYPES = { + 'custom' => :custom, + 'shell' => :shell, + 'docker' => :docker, + 'docker-windows' => :docker_windows, + 'docker-ssh' => :docker_ssh, + 'ssh' => :ssh, + 'parallels' => :parallels, + 'virtualbox' => :virtualbox, + 'docker+machine' => :docker_machine, + 'docker-ssh+machine' => :docker_ssh_machine, + 'kubernetes' => :kubernetes + }.freeze + def cleanup_runner_queue Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index 52a31863fb2..82390ccc538 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -7,7 +7,6 @@ module Ci self.limit_name = 'ci_registered_group_runners' self.limit_scope = :group self.limit_relation = :recent_runners - self.limit_feature_flag_for_override = :ci_runner_limits_override belongs_to :runner, inverse_of: :runner_namespaces belongs_to :namespace, inverse_of: :runner_namespaces, class_name: '::Namespace' diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index 148a29a0f8b..42c24c8c8d1 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -7,7 +7,6 @@ module Ci self.limit_name = 'ci_registered_project_runners' self.limit_scope = :project self.limit_relation = :recent_runners - self.limit_feature_flag_for_override = :ci_runner_limits_override belongs_to :runner, inverse_of: :runner_projects belongs_to :project, inverse_of: :runner_projects diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index e2b15497638..8c4e97ac840 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -22,6 +22,7 @@ module Ci scope :ordered, -> { order(position: :asc) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } scope :by_name, ->(names) { where(name: names) } + scope :by_position, ->(positions) { where(position: positions) } with_options unless: :importing? do validates :project, presence: true |