diff options
Diffstat (limited to 'app/models/ci')
-rw-r--r-- | app/models/ci/bridge.rb | 2 | ||||
-rw-r--r-- | app/models/ci/build.rb | 72 | ||||
-rw-r--r-- | app/models/ci/build_metadata.rb | 4 | ||||
-rw-r--r-- | app/models/ci/build_need.rb | 2 | ||||
-rw-r--r-- | app/models/ci/build_runner_session.rb | 2 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunk.rb | 2 | ||||
-rw-r--r-- | app/models/ci/build_trace_metadata.rb | 11 | ||||
-rw-r--r-- | app/models/ci/job_token/project_scope_link.rb | 2 | ||||
-rw-r--r-- | app/models/ci/job_token/scope.rb | 11 | ||||
-rw-r--r-- | app/models/ci/pipeline.rb | 31 | ||||
-rw-r--r-- | app/models/ci/processable.rb | 3 | ||||
-rw-r--r-- | app/models/ci/resource_group.rb | 32 | ||||
-rw-r--r-- | app/models/ci/runner.rb | 26 | ||||
-rw-r--r-- | app/models/ci/runner_namespace.rb | 1 | ||||
-rw-r--r-- | app/models/ci/runner_project.rb | 1 | ||||
-rw-r--r-- | app/models/ci/stage.rb | 2 |
16 files changed, 135 insertions, 69 deletions
diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 97fb8233d34..50bda64d537 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -31,7 +31,7 @@ module Ci next unless bridge.triggers_downstream_pipeline? bridge.run_after_commit do - ::Ci::CreateCrossProjectPipelineWorker.perform_async(bridge.id) + ::Ci::CreateDownstreamPipelineWorker.perform_async(bridge.id) end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index e2e24247679..990ef71a457 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -42,6 +42,10 @@ module Ci has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build + # Projects::DestroyService destroys Ci::Pipelines, which use_fast_destroy on :job_artifacts + # before we delete builds. By doing this, the relation should be empty and not fire any + # DELETE queries when the Ci::Build is destroyed. The next step is to remove `dependent: :destroy`. + # Details: https://gitlab.com/gitlab-org/gitlab/-/issues/24644#note_689472685 has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id @@ -55,6 +59,8 @@ module Ci has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build + has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', dependent: :nullify, inverse_of: :build, foreign_key: :ci_build_id # rubocop:disable Cop/ActiveRecordDependent + accepts_nested_attributes_for :runner_session, update_only: true accepts_nested_attributes_for :job_variables @@ -64,8 +70,8 @@ module Ci delegate :gitlab_deploy_token, to: :project delegate :trigger_short_token, to: :trigger_request, allow_nil: true - ignore_columns :id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22' - ignore_columns :stage_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22' + ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' + ignore_columns :stage_id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' ## # Since Gitlab 11.5, deployments records started being created right after @@ -192,7 +198,6 @@ module Ci add_authentication_token_field :token, encrypted: :required before_save :ensure_token - before_destroy { unscoped_project } after_save :stick_build_if_status_changed @@ -308,8 +313,10 @@ module Ci end after_transition pending: :running do |build| - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - build.deployment&.run + unless build.update_deployment_after_transaction_commit? + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment&.run + end end build.run_after_commit do @@ -332,8 +339,10 @@ module Ci end after_transition any => [:success] do |build| - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - build.deployment&.succeed + unless build.update_deployment_after_transaction_commit? + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment&.succeed + end end build.run_after_commit do @@ -346,12 +355,14 @@ module Ci next unless build.project next unless build.deployment - begin - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - build.deployment.drop! + unless build.update_deployment_after_transaction_commit? + begin + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment.drop! + end + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id) end - rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id) end true @@ -370,14 +381,29 @@ module Ci end after_transition any => [:skipped, :canceled] do |build, transition| - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - if transition.to_name == :skipped - build.deployment&.skip - else - build.deployment&.cancel + unless build.update_deployment_after_transaction_commit? + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + if transition.to_name == :skipped + build.deployment&.skip + else + build.deployment&.cancel + end end end end + + # Synchronize Deployment Status + # Please note that the data integirty is not assured because we can't use + # a database transaction due to DB decomposition. + after_transition do |build, transition| + next if transition.loopback? + next unless build.project + next unless build.update_deployment_after_transaction_commit? + + build.run_after_commit do + build.deployment&.sync_status_with(build) + end + end end def self.build_matchers(project) @@ -1094,6 +1120,12 @@ module Ci runner&.instance_type? end + def update_deployment_after_transaction_commit? + strong_memoize(:update_deployment_after_transaction_commit) do + Feature.enabled?(:update_deployment_after_transaction_commit, project, default_enabled: :yaml) + end + end + protected def run_status_commit_hooks! @@ -1108,7 +1140,7 @@ module Ci return unless saved_change_to_status? return unless running? - ::Gitlab::Database::LoadBalancing::Sticking.stick(:build, id) + self.class.sticking.stick(:build, id) end def status_commit_hooks @@ -1154,10 +1186,6 @@ module Ci self.update(erased_by: user, erased_at: Time.current, artifacts_expire_at: nil) end - def unscoped_project - @unscoped_project ||= Project.unscoped.find_by(id: project_id) - end - def environment_url options&.dig(:environment, :url) || persisted_environment&.external_url end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 90237a4be52..0d6d6f7a6a5 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -37,8 +37,8 @@ module Ci job_timeout_source: 4 } - ignore_column :build_id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22' - ignore_columns :id_convert_to_bigint, remove_with: '14.3', remove_after: '2021-09-22' + ignore_column :build_id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' + ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' def update_timeout_state timeout = timeout_with_highest_precedence diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index 003659570b3..bf1470ca20f 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -5,8 +5,6 @@ module Ci include BulkInsertSafe include IgnorableColumns - ignore_columns :build_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22' - belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs validates :build, presence: true diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb index 45de47116cd..e12c0f82c99 100644 --- a/app/models/ci/build_runner_session.rb +++ b/app/models/ci/build_runner_session.rb @@ -6,8 +6,6 @@ module Ci class BuildRunnerSession < Ci::ApplicationRecord include IgnorableColumns - ignore_columns :build_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22' - TERMINAL_SUBPROTOCOL = 'terminal.gitlab.com' DEFAULT_SERVICE_NAME = 'build' DEFAULT_PORT_NAME = 'default_port' diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 7a15d7ba940..6edb5ef4579 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -9,8 +9,6 @@ module Ci include ::Gitlab::OptimisticLocking include IgnorableColumns - ignore_columns :build_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22' - belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id default_value_for :data_store, :redis_trace_chunks diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 901b84ceec6..1ffa0e31f99 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -37,8 +37,10 @@ module Ci increment!(:archival_attempts, touch: :last_archival_attempt_at) end - def track_archival!(trace_artifact_id) - update!(trace_artifact_id: trace_artifact_id, archived_at: Time.current) + def track_archival!(trace_artifact_id, checksum) + update!(trace_artifact_id: trace_artifact_id, + checksum: checksum, + archived_at: Time.current) end def archival_attempts_message @@ -49,6 +51,11 @@ module Ci end end + def remote_checksum_valid? + checksum.present? && + checksum == remote_checksum + end + private def backoff diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb index 99118f8090b..c2ab8ca0929 100644 --- a/app/models/ci/job_token/project_scope_link.rb +++ b/app/models/ci/job_token/project_scope_link.rb @@ -5,7 +5,7 @@ module Ci module JobToken - class ProjectScopeLink < ApplicationRecord + class ProjectScopeLink < Ci::ApplicationRecord self.table_name = 'ci_job_token_project_scope_links' belongs_to :source_project, class_name: 'Project' diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb index 42cfdc21d66..3a5765aa00c 100644 --- a/app/models/ci/job_token/scope.rb +++ b/app/models/ci/job_token/scope.rb @@ -32,12 +32,15 @@ module Ci def all_projects Project.from_union([ Project.id_in(source_project), - Project.where_exists( - Ci::JobToken::ProjectScopeLink - .from_project(source_project) - .where('projects.id = ci_job_token_project_scope_links.target_project_id')) + Project.id_in(target_project_ids) ], remove_duplicates: false) end + + private + + def target_project_ids + Ci::JobToken::ProjectScopeLink.from_project(source_project).pluck(:target_project_id) + end end end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 1a0cec3c935..0041ec5135c 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -82,7 +82,8 @@ module Ci # Merge requests for which the current pipeline is running against # the merge request's latest commit. has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest' - + has_many :package_build_infos, class_name: 'Packages::BuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent + has_many :package_file_build_infos, class_name: 'Packages::PackageFileBuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline @@ -861,11 +862,6 @@ module Ci self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self) end - def execute_hooks - project.execute_hooks(pipeline_data, :pipeline_hooks) if project.has_active_hooks?(:pipeline_hooks) - project.execute_integrations(pipeline_data, :pipeline_hooks) if project.has_active_integrations?(:pipeline_hooks) - end - # All the merge requests for which the current pipeline runs/ran against def all_merge_requests @all_merge_requests ||= @@ -929,9 +925,22 @@ module Ci end def environments_in_self_and_descendants - environment_ids = self_and_descendants.joins(:deployments).select(:'deployments.environment_id') + if ::Feature.enabled?(:avoid_cross_joins_environments_in_self_and_descendants, default_enabled: :yaml) + # We limit to 100 unique environments for application safety. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 + expanded_environment_names = + builds_in_self_and_descendants.joins(:metadata) + .where.not('ci_builds_metadata.expanded_environment_name' => nil) + .distinct('ci_builds_metadata.expanded_environment_name') + .limit(100) + .pluck(:expanded_environment_name) + + Environment.where(project: project, name: expanded_environment_names) + else + environment_ids = self_and_descendants.joins(:deployments).select(:'deployments.environment_id') - Environment.where(id: environment_ids) + Environment.where(id: environment_ids) + end end # With multi-project and parent-child pipelines @@ -1251,12 +1260,6 @@ module Ci messages.build(severity: severity, content: content) end - def pipeline_data - strong_memoize(:pipeline_data) do - Gitlab::DataBuilder::Pipeline.build(self) - end - end - def merge_request_diff_sha return unless merge_request? diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 30d335fd7d5..372df8cc264 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -58,7 +58,8 @@ module Ci after_transition any => ::Ci::Processable.completed_statuses do |processable| next unless processable.with_resource_group? - next unless processable.resource_group.release_resource_from(processable) + + processable.resource_group.release_resource_from(processable) processable.run_after_commit do Ci::ResourceGroups::AssignResourceFromResourceGroupWorker diff --git a/app/models/ci/resource_group.rb b/app/models/ci/resource_group.rb index 8a7456041e6..6d25f747a9d 100644 --- a/app/models/ci/resource_group.rb +++ b/app/models/ci/resource_group.rb @@ -14,6 +14,12 @@ module Ci before_create :ensure_resource + enum process_mode: { + unordered: 0, + oldest_first: 1, + newest_first: 2 + } + ## # NOTE: This is concurrency-safe method that the subquery in the `UPDATE` # works as explicit locking. @@ -25,8 +31,34 @@ module Ci resources.retained_by(processable).update_all(build_id: nil) > 0 end + def upcoming_processables + if unordered? + processables.waiting_for_resource + elsif oldest_first? + processables.waiting_for_resource_or_upcoming + .order(Arel.sql("commit_id ASC, #{sort_by_job_status}")) + elsif newest_first? + processables.waiting_for_resource_or_upcoming + .order(Arel.sql("commit_id DESC, #{sort_by_job_status}")) + else + Ci::Processable.none + end + end + private + # In order to avoid deadlock, we do NOT specify the job execution order in the same pipeline. + # The system processes wherever ready to transition to `pending` status from `waiting_for_resource`. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/202186 for more information. + def sort_by_job_status + <<~SQL + CASE status + WHEN 'waiting_for_resource' THEN 0 + ELSE 1 + END ASC + SQL + end + def ensure_resource # Currently we only support one resource per group, which means # maximum one build can be set to the resource group, thus builds diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 4aa232ad26b..2f718ad7582 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -51,7 +51,7 @@ module Ci has_many :runner_projects, inverse_of: :runner, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects has_many :runner_namespaces, inverse_of: :runner, autosave: true - has_many :groups, through: :runner_namespaces + has_many :groups, through: :runner_namespaces, disable_joins: true has_one :last_build, -> { order('id DESC') }, class_name: 'Ci::Build' @@ -246,7 +246,7 @@ module Ci begin transaction do - self.projects << project + self.runner_projects << ::Ci::RunnerProject.new(project: project, runner: self) self.save! end rescue ActiveRecord::RecordInvalid => e @@ -280,7 +280,7 @@ module Ci end def belongs_to_more_than_one_project? - self.projects.limit(2).count(:all) > 1 + runner_projects.limit(2).count(:all) > 1 end def assigned_to_group? @@ -309,7 +309,9 @@ module Ci end def only_for?(project) - projects == [project] + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do + projects == [project] + end end def short_sha @@ -344,7 +346,7 @@ module Ci # intention here is not to execute `Ci::RegisterJobService#execute` on # the primary database. # - ::Gitlab::Database::LoadBalancing::Sticking.stick(:runner, id) + ::Ci::Runner.sticking.stick(:runner, id) SecureRandom.hex.tap do |new_update| ::Gitlab::Workhorse.set_key_and_notify(runner_queue_key, new_update, @@ -428,10 +430,8 @@ module Ci end def no_projects - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do - if projects.any? - errors.add(:runner, 'cannot have projects assigned') - end + if runner_projects.any? + errors.add(:runner, 'cannot have projects assigned') end end @@ -444,14 +444,16 @@ module Ci end def any_project - unless projects.any? + unless runner_projects.any? errors.add(:runner, 'needs to be assigned to at least one project') end end def exactly_one_group - unless groups.one? - errors.add(:runner, 'needs to be assigned to exactly one group') + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do + unless groups.one? + errors.add(:runner, 'needs to be assigned to exactly one group') + end end end diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index d1353b97ed9..52a31863fb2 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -7,7 +7,6 @@ module Ci self.limit_name = 'ci_registered_group_runners' self.limit_scope = :group self.limit_relation = :recent_runners - self.limit_feature_flag = :ci_runner_limits self.limit_feature_flag_for_override = :ci_runner_limits_override belongs_to :runner, inverse_of: :runner_namespaces diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index e1c435e9b1f..148a29a0f8b 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -7,7 +7,6 @@ module Ci self.limit_name = 'ci_registered_project_runners' self.limit_scope = :project self.limit_relation = :recent_runners - self.limit_feature_flag = :ci_runner_limits self.limit_feature_flag_for_override = :ci_runner_limits_override belongs_to :runner, inverse_of: :runner_projects diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 39e26bf2785..131e18adf62 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -8,8 +8,6 @@ module Ci include Presentable include IgnorableColumns - ignore_column :id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22' - enum status: Ci::HasStatus::STATUSES_ENUM belongs_to :project |