diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 13:18:24 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 13:18:24 +0000 |
commit | 0653e08efd039a5905f3fa4f6e9cef9f5d2f799c (patch) | |
tree | 4dcc884cf6d81db44adae4aa99f8ec1233a41f55 /app/services/ci | |
parent | 744144d28e3e7fddc117924fef88de5d9674fe4c (diff) | |
download | gitlab-ce-0653e08efd039a5905f3fa4f6e9cef9f5d2f799c.tar.gz |
Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42
Diffstat (limited to 'app/services/ci')
-rw-r--r-- | app/services/ci/after_requeue_job_service.rb | 13 | ||||
-rw-r--r-- | app/services/ci/archive_trace_service.rb | 9 | ||||
-rw-r--r-- | app/services/ci/drop_pipeline_service.rb | 8 | ||||
-rw-r--r-- | app/services/ci/external_pull_requests/create_pipeline_service.rb | 10 | ||||
-rw-r--r-- | app/services/ci/pipeline_schedules/calculate_next_run_service.rb | 1 | ||||
-rw-r--r-- | app/services/ci/pipelines/add_job_service.rb | 4 | ||||
-rw-r--r-- | app/services/ci/queue/build_queue_service.rb | 36 | ||||
-rw-r--r-- | app/services/ci/queue/builds_table_strategy.rb | 8 | ||||
-rw-r--r-- | app/services/ci/queue/pending_builds_strategy.rb | 22 | ||||
-rw-r--r-- | app/services/ci/register_job_service.rb | 63 | ||||
-rw-r--r-- | app/services/ci/stuck_builds/drop_helpers.rb | 58 | ||||
-rw-r--r-- | app/services/ci/stuck_builds/drop_service.rb | 62 | ||||
-rw-r--r-- | app/services/ci/update_build_queue_service.rb | 14 | ||||
-rw-r--r-- | app/services/ci/update_pending_build_service.rb | 45 |
14 files changed, 286 insertions, 67 deletions
diff --git a/app/services/ci/after_requeue_job_service.rb b/app/services/ci/after_requeue_job_service.rb index f717dd0862c..9101ae91967 100644 --- a/app/services/ci/after_requeue_job_service.rb +++ b/app/services/ci/after_requeue_job_service.rb @@ -10,16 +10,9 @@ module Ci private def process_subsequent_jobs(processable) - if Feature.enabled?(:ci_same_stage_job_needs, processable.project, default_enabled: :yaml) - (stage_dependent_jobs(processable) | needs_dependent_jobs(processable)) - .each do |processable| - process(processable) - end - else - skipped_jobs(processable).after_stage(processable.stage_idx) - .find_each do |job| - process(job) - end + (stage_dependent_jobs(processable) | needs_dependent_jobs(processable)) + .each do |processable| + process(processable) end end diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index bc3219fbd79..995b58c6882 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -3,6 +3,13 @@ module Ci class ArchiveTraceService def execute(job, worker_name:) + unless job.trace.can_attempt_archival_now? + Sidekiq.logger.warn(class: worker_name, + message: job.trace.archival_attempts_message, + job_id: job.id) + return + end + # TODO: Remove this logging once we confirmed new live trace architecture is functional. # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. unless job.has_live_trace? @@ -25,6 +32,8 @@ module Ci rescue ::Gitlab::Ci::Trace::AlreadyArchivedError # It's already archived, thus we can safely ignore this exception. rescue StandardError => e + job.trace.increment_archival_attempts! + # Tracks this error with application logs, Sentry, and Prometheus. # If `archive!` keeps failing for over a week, that could incur data loss. # (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture) diff --git a/app/services/ci/drop_pipeline_service.rb b/app/services/ci/drop_pipeline_service.rb index 16d3abcbfa0..5772ab8f29c 100644 --- a/app/services/ci/drop_pipeline_service.rb +++ b/app/services/ci/drop_pipeline_service.rb @@ -2,8 +2,7 @@ module Ci class DropPipelineService - PRELOADED_COMMIT_STATUS_RELATIONS = [:project, :pipeline].freeze - PRELOADED_CI_BUILD_RELATIONS = [:metadata, :deployment, :taggings].freeze + PRELOADED_RELATIONS = [:project, :pipeline, :metadata, :deployment, :taggings].freeze # execute service asynchronously for each cancelable pipeline def execute_async_for_all(pipelines, failure_reason, context_user) @@ -30,11 +29,8 @@ module Ci private - # rubocop: disable CodeReuse/ActiveRecord def preload_associations_for_drop(commit_status_batch) - ActiveRecord::Associations::Preloader.new.preload(commit_status_batch, PRELOADED_COMMIT_STATUS_RELATIONS) - ActiveRecord::Associations::Preloader.new.preload(commit_status_batch.select { |job| job.is_a?(Ci::Build) }, PRELOADED_CI_BUILD_RELATIONS) + Preloaders::CommitStatusPreloader.new(commit_status_batch).execute(PRELOADED_RELATIONS) end - # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/app/services/ci/external_pull_requests/create_pipeline_service.rb b/app/services/ci/external_pull_requests/create_pipeline_service.rb index 83499524a8e..dd93ca4708e 100644 --- a/app/services/ci/external_pull_requests/create_pipeline_service.rb +++ b/app/services/ci/external_pull_requests/create_pipeline_service.rb @@ -16,8 +16,14 @@ module Ci private def create_pipeline_for(pull_request) - Ci::CreatePipelineService.new(project, current_user, create_params(pull_request)) - .execute(:external_pull_request_event, external_pull_request: pull_request) + if ::Feature.enabled?(:ci_create_external_pr_pipeline_async, project, default_enabled: :yaml) + Ci::ExternalPullRequests::CreatePipelineWorker.perform_async( + project.id, current_user.id, pull_request.id + ) + else + Ci::CreatePipelineService.new(project, current_user, create_params(pull_request)) + .execute(:external_pull_request_event, external_pull_request: pull_request) + end end def create_params(pull_request) diff --git a/app/services/ci/pipeline_schedules/calculate_next_run_service.rb b/app/services/ci/pipeline_schedules/calculate_next_run_service.rb index 9c8f6b47288..6c8ccb017e9 100644 --- a/app/services/ci/pipeline_schedules/calculate_next_run_service.rb +++ b/app/services/ci/pipeline_schedules/calculate_next_run_service.rb @@ -8,7 +8,6 @@ module Ci def execute(schedule, fallback_method:) @schedule = schedule - return fallback_method.call unless ::Feature.enabled?(:ci_daily_limit_for_pipeline_schedules, project, default_enabled: :yaml) return fallback_method.call unless plan_cron&.cron_valid? now = Time.zone.now diff --git a/app/services/ci/pipelines/add_job_service.rb b/app/services/ci/pipelines/add_job_service.rb index 41f9903e48c..53536b6fdf9 100644 --- a/app/services/ci/pipelines/add_job_service.rb +++ b/app/services/ci/pipelines/add_job_service.rb @@ -21,14 +21,14 @@ module Ci Ci::Pipeline.transaction do yield(job) - job.update_older_statuses_retried! if Feature.enabled?(:ci_fix_commit_status_retried, pipeline.project, default_enabled: :yaml) + job.update_older_statuses_retried! end end else Ci::Pipeline.transaction do yield(job) - job.update_older_statuses_retried! if Feature.enabled?(:ci_fix_commit_status_retried, pipeline.project, default_enabled: :yaml) + job.update_older_statuses_retried! end end diff --git a/app/services/ci/queue/build_queue_service.rb b/app/services/ci/queue/build_queue_service.rb index 99408d529b2..3276c427923 100644 --- a/app/services/ci/queue/build_queue_service.rb +++ b/app/services/ci/queue/build_queue_service.rb @@ -24,26 +24,30 @@ module Ci # rubocop:disable CodeReuse/ActiveRecord def builds_for_group_runner - # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL` - groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces) + if strategy.use_denormalized_namespace_traversal_ids? + strategy.builds_for_group_runner + else + # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL` + groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces) - hierarchy_groups = Gitlab::ObjectHierarchy - .new(groups) - .base_and_descendants + hierarchy_groups = Gitlab::ObjectHierarchy + .new(groups) + .base_and_descendants - projects = Project.where(namespace_id: hierarchy_groups) - .with_group_runners_enabled - .with_builds_enabled - .without_deleted + projects = Project.where(namespace_id: hierarchy_groups) + .with_group_runners_enabled + .with_builds_enabled + .without_deleted - relation = new_builds.where(project: projects) + relation = new_builds.where(project: projects) - order(relation) + order(relation) + end end def builds_for_project_runner relation = new_builds - .where(project: runner.projects.without_deleted.with_builds_enabled) + .where(project: runner_projects_relation) order(relation) end @@ -83,6 +87,14 @@ module Ci end end end + + def runner_projects_relation + if ::Feature.enabled?(:ci_pending_builds_project_runners_decoupling, runner, default_enabled: :yaml) + runner.runner_projects.select(:project_id) + else + runner.projects.without_deleted.with_builds_enabled + end + end end end end diff --git a/app/services/ci/queue/builds_table_strategy.rb b/app/services/ci/queue/builds_table_strategy.rb index d0a343cb9d4..ac449a5289e 100644 --- a/app/services/ci/queue/builds_table_strategy.rb +++ b/app/services/ci/queue/builds_table_strategy.rb @@ -31,6 +31,10 @@ module Ci end end + def builds_for_group_runner + raise NotImplementedError + end + def builds_matching_tag_ids(relation, ids) # pick builds that does not have other tags than runner's one relation.matches_tag_ids(ids) @@ -61,6 +65,10 @@ module Ci false end + def use_denormalized_namespace_traversal_ids? + false + end + private def running_builds_for_shared_runners diff --git a/app/services/ci/queue/pending_builds_strategy.rb b/app/services/ci/queue/pending_builds_strategy.rb index efe3a981d3a..7a913e47df4 100644 --- a/app/services/ci/queue/pending_builds_strategy.rb +++ b/app/services/ci/queue/pending_builds_strategy.rb @@ -16,12 +16,26 @@ module Ci builds_ordered_for_shared_runners(shared_builds) end + def builds_for_group_runner + return new_builds.none if runner.namespace_ids.empty? + + new_builds.where('ci_pending_builds.namespace_traversal_ids && ARRAY[?]::int[]', runner.namespace_ids) + end + def builds_matching_tag_ids(relation, ids) - relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id')) + if ::Feature.enabled?(:ci_queueing_denormalize_tags_information, runner, default_enabled: :yaml) + relation.for_tags(runner.tags_ids) + else + relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id')) + end end def builds_with_any_tags(relation) - relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id')) + if ::Feature.enabled?(:ci_queueing_denormalize_tags_information, runner, default_enabled: :yaml) + relation.where('cardinality(tag_ids) > 0') + else + relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id')) + end end def order(relation) @@ -44,6 +58,10 @@ module Ci ::Feature.enabled?(:ci_queueing_denormalize_ci_minutes_information, runner, type: :development, default_enabled: :yaml) end + def use_denormalized_namespace_traversal_ids? + ::Feature.enabled?(:ci_queueing_denormalize_namespace_traversal_ids, runner, type: :development, default_enabled: :yaml) + end + private def builds_available_for_shared_runners diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index dc046e1d164..c46ddd22558 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -103,40 +103,42 @@ module Ci # rubocop: disable CodeReuse/ActiveRecord def each_build(params, &blk) - queue = ::Ci::Queue::BuildQueueService.new(runner) - - builds = begin - if runner.instance_type? - queue.builds_for_shared_runner - elsif runner.group_type? - queue.builds_for_group_runner - else - queue.builds_for_project_runner + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339429') do + queue = ::Ci::Queue::BuildQueueService.new(runner) + + builds = begin + if runner.instance_type? + queue.builds_for_shared_runner + elsif runner.group_type? + queue.builds_for_group_runner + else + queue.builds_for_project_runner + end end - end - if runner.ref_protected? - builds = queue.builds_for_protected_runner(builds) - end + if runner.ref_protected? + builds = queue.builds_for_protected_runner(builds) + end - # pick builds that does not have other tags than runner's one - builds = queue.builds_matching_tag_ids(builds, runner.tags.ids) + # pick builds that does not have other tags than runner's one + builds = queue.builds_matching_tag_ids(builds, runner.tags.ids) - # pick builds that have at least one tag - unless runner.run_untagged? - builds = queue.builds_with_any_tags(builds) - end + # pick builds that have at least one tag + unless runner.run_untagged? + builds = queue.builds_with_any_tags(builds) + end - # pick builds that older than specified age - if params.key?(:job_age) - builds = queue.builds_queued_before(builds, params[:job_age].seconds.ago) - end + # pick builds that older than specified age + if params.key?(:job_age) + builds = queue.builds_queued_before(builds, params[:job_age].seconds.ago) + end - build_ids = retrieve_queue(-> { queue.execute(builds) }) + build_ids = retrieve_queue(-> { queue.execute(builds) }) - @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type) + @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type) - build_ids.each { |build_id| yield Ci::Build.find(build_id) } + build_ids.each { |build_id| yield Ci::Build.find(build_id) } + end end # rubocop: enable CodeReuse/ActiveRecord @@ -269,6 +271,15 @@ module Ci missing_dependency_failure: -> (build, _) { !build.has_valid_build_dependencies? }, runner_unsupported: -> (build, params) { !build.supported_runner?(params.dig(:info, :features)) }, archived_failure: -> (build, _) { build.archived? } + }.merge(builds_enabled_checks) + end + + def builds_enabled_checks + return {} unless ::Feature.enabled?(:ci_queueing_builds_enabled_checks, runner, default_enabled: :yaml) + + { + project_deleted: -> (build, _) { build.project.pending_delete? }, + builds_disabled: -> (build, _) { !build.project.builds_enabled? } } end end diff --git a/app/services/ci/stuck_builds/drop_helpers.rb b/app/services/ci/stuck_builds/drop_helpers.rb new file mode 100644 index 00000000000..f79b805c23d --- /dev/null +++ b/app/services/ci/stuck_builds/drop_helpers.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Ci + module StuckBuilds + module DropHelpers + def drop(builds, failure_reason:) + fetch(builds) do |build| + drop_build :outdated, build, failure_reason + end + end + + def drop_stuck(builds, failure_reason:) + fetch(builds) do |build| + break unless build.stuck? + + drop_build :stuck, build, failure_reason + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def fetch(builds) + loop do + jobs = builds.includes(:tags, :runner, project: [:namespace, :route]) + .limit(100) + .to_a + + break if jobs.empty? + + jobs.each do |job| + Gitlab::ApplicationContext.with_context(project: job.project) { yield(job) } + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def drop_build(type, build, reason) + Gitlab::AppLogger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{build.status}, failure_reason: #{reason})" + Gitlab::OptimisticLocking.retry_lock(build, 3, name: 'stuck_ci_jobs_worker_drop_build') do |b| + b.drop(reason) + end + rescue StandardError => ex + build.doom! + + track_exception_for_build(ex, build) + end + + def track_exception_for_build(ex, build) + Gitlab::ErrorTracking.track_exception(ex, + build_id: build.id, + build_name: build.name, + build_stage: build.stage, + pipeline_id: build.pipeline_id, + project_id: build.project_id + ) + end + end + end +end diff --git a/app/services/ci/stuck_builds/drop_service.rb b/app/services/ci/stuck_builds/drop_service.rb new file mode 100644 index 00000000000..3fee9a94381 --- /dev/null +++ b/app/services/ci/stuck_builds/drop_service.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Ci + module StuckBuilds + class DropService + include DropHelpers + + BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour + BUILD_PENDING_OUTDATED_TIMEOUT = 1.day + BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour + BUILD_PENDING_STUCK_TIMEOUT = 1.hour + BUILD_LOOKBACK = 5.days + + def execute + Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds" + + drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure) + + drop( + pending_builds(BUILD_PENDING_OUTDATED_TIMEOUT.ago), + failure_reason: :stuck_or_timeout_failure + ) + + drop(scheduled_timed_out_builds, failure_reason: :stale_schedule) + + drop_stuck( + pending_builds(BUILD_PENDING_STUCK_TIMEOUT.ago), + failure_reason: :stuck_or_timeout_failure + ) + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + # We're adding the ordering clause by `created_at` and `project_id` + # because we want to force the query planner to use the + # `ci_builds_gitlab_monitor_metrics` index all the time. + def pending_builds(timeout) + if Feature.enabled?(:ci_new_query_for_pending_stuck_jobs) + Ci::Build.pending.created_at_before(timeout).updated_at_before(timeout).order(created_at: :asc, project_id: :asc) + else + Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: timeout) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def scheduled_timed_out_builds + Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord + 'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?', + BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago + ) + end + + def running_timed_out_builds + Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord + 'ci_builds.updated_at < ?', + BUILD_RUNNING_OUTDATED_TIMEOUT.ago + ) + end + end + end +end diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb index eea09e9ac67..c1cbf031ca1 100644 --- a/app/services/ci/update_build_queue_service.rb +++ b/app/services/ci/update_build_queue_service.rb @@ -99,15 +99,17 @@ module Ci private def tick_for(build, runners) - runners = runners.with_recent_runner_queue - runners = runners.with_tags if Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml) + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do + runners = runners.with_recent_runner_queue + runners = runners.with_tags if Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml) - metrics.observe_active_runners(-> { runners.to_a.size }) + metrics.observe_active_runners(-> { runners.to_a.size }) - runners.each do |runner| - metrics.increment_runner_tick(runner) + runners.each do |runner| + metrics.increment_runner_tick(runner) - runner.pick_build!(build) + runner.pick_build!(build) + end end end diff --git a/app/services/ci/update_pending_build_service.rb b/app/services/ci/update_pending_build_service.rb new file mode 100644 index 00000000000..dcba06e60bf --- /dev/null +++ b/app/services/ci/update_pending_build_service.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Ci + class UpdatePendingBuildService + VALID_PARAMS = %i[instance_runners_enabled].freeze + + InvalidParamsError = Class.new(StandardError) + InvalidModelError = Class.new(StandardError) + + def initialize(model, update_params) + @model = model + @update_params = update_params + + validations! + end + + def execute + return unless ::Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, @model, default_enabled: :yaml) + + @model.pending_builds.each_batch do |relation| + relation.update_all(@update_params) + end + end + + private + + def validations! + validate_model! && validate_params! + end + + def validate_model! + raise InvalidModelError unless @model.is_a?(::Project) || @model.is_a?(::Group) + + true + end + + def validate_params! + extra_params = @update_params.keys - VALID_PARAMS + + raise InvalidParamsError, "Unvalid params: #{extra_params.join(', ')}" unless extra_params.empty? + + true + end + end +end |