diff options
Diffstat (limited to 'app/services/ci')
| -rw-r--r-- | app/services/ci/create_downstream_pipeline_service.rb | 4 | ||||
| -rw-r--r-- | app/services/ci/create_pipeline_service.rb | 16 | ||||
| -rw-r--r-- | app/services/ci/create_web_ide_terminal_service.rb | 2 | ||||
| -rw-r--r-- | app/services/ci/destroy_expired_job_artifacts_service.rb | 57 | ||||
| -rw-r--r-- | app/services/ci/pipeline_artifacts/coverage_report_service.rb (renamed from app/services/ci/pipelines/create_artifact_service.rb) | 4 | ||||
| -rw-r--r-- | app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb | 52 | ||||
| -rw-r--r-- | app/services/ci/pipeline_trigger_service.rb | 8 | ||||
| -rw-r--r-- | app/services/ci/play_build_service.rb | 4 | ||||
| -rw-r--r-- | app/services/ci/process_build_service.rb | 21 | ||||
| -rw-r--r-- | app/services/ci/register_job_service.rb | 4 | ||||
| -rw-r--r-- | app/services/ci/retry_build_service.rb | 6 | ||||
| -rw-r--r-- | app/services/ci/retry_pipeline_service.rb | 2 | ||||
| -rw-r--r-- | app/services/ci/test_failure_history_service.rb | 1 | ||||
| -rw-r--r-- | app/services/ci/update_build_state_service.rb | 2 |
14 files changed, 134 insertions, 49 deletions
diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb index 86d0cf079fc..629d85b041f 100644 --- a/app/services/ci/create_downstream_pipeline_service.rb +++ b/app/services/ci/create_downstream_pipeline_service.rb @@ -33,9 +33,7 @@ module Ci pipeline_params.fetch(:target_revision)) downstream_pipeline = service.execute( - pipeline_params.fetch(:source), **pipeline_params[:execute_params]) do |pipeline| - pipeline.variables.build(@bridge.downstream_variables) - end + pipeline_params.fetch(:source), **pipeline_params[:execute_params]) downstream_pipeline.tap do |pipeline| update_bridge_status!(@bridge, pipeline) diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index dbe81521cfc..d3001e54288 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -27,6 +27,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Limit::JobActivity, Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines, Gitlab::Ci::Pipeline::Chain::Metrics, + Gitlab::Ci::Pipeline::Chain::TemplateUsage, Gitlab::Ci::Pipeline::Chain::Pipeline::Process].freeze # Create a new pipeline in the specified project. @@ -81,7 +82,11 @@ module Ci .new(pipeline, command, SEQUENCE) .build! - schedule_head_pipeline_update if pipeline.persisted? + if pipeline.persisted? + schedule_head_pipeline_update + record_conversion_event + create_namespace_onboarding_action + end # If pipeline is not persisted, try to recover IID pipeline.reset_project_iid unless pipeline.persisted? @@ -116,6 +121,15 @@ module Ci end end + def record_conversion_event + Experiments::RecordConversionEventWorker.perform_async(:ci_syntax_templates, current_user.id) + Experiments::RecordConversionEventWorker.perform_async(:pipelines_empty_state, current_user.id) + end + + def create_namespace_onboarding_action + Namespaces::OnboardingPipelineCreatedWorker.perform_async(project.namespace_id) + end + def extra_options(content: nil, dry_run: false) { content: content, dry_run: dry_run } end diff --git a/app/services/ci/create_web_ide_terminal_service.rb b/app/services/ci/create_web_ide_terminal_service.rb index a78281aed16..785d82094b9 100644 --- a/app/services/ci/create_web_ide_terminal_service.rb +++ b/app/services/ci/create_web_ide_terminal_service.rb @@ -6,7 +6,7 @@ module Ci TerminalCreationError = Class.new(StandardError) - TERMINAL_NAME = 'terminal'.freeze + TERMINAL_NAME = 'terminal' attr_reader :terminal diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb index 6e7caba8545..7d8a3c17abe 100644 --- a/app/services/ci/destroy_expired_job_artifacts_service.rb +++ b/app/services/ci/destroy_expired_job_artifacts_service.rb @@ -12,6 +12,10 @@ module Ci EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock' LOCK_TIMEOUT = 6.minutes + def initialize + @removed_artifacts_count = 0 + end + ## # Destroy expired job artifacts on GitLab instance # @@ -20,40 +24,22 @@ module Ci # which is scheduled every 7 minutes. def execute in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do - loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do - destroy_artifacts_batch - end + destroy_job_artifacts_with_slow_iteration(Time.current) end - end - - private - def destroy_artifacts_batch - destroy_job_artifacts_batch || destroy_pipeline_artifacts_batch + @removed_artifacts_count end - def destroy_job_artifacts_batch - artifacts = Ci::JobArtifact - .expired(BATCH_SIZE) - .unlocked - .with_destroy_preloads - .to_a - - return false if artifacts.empty? - - parallel_destroy_batch(artifacts) - true - end - - # TODO: Make sure this can also be parallelized - # https://gitlab.com/gitlab-org/gitlab/-/issues/270973 - def destroy_pipeline_artifacts_batch - artifacts = Ci::PipelineArtifact.expired(BATCH_SIZE).to_a - return false if artifacts.empty? + private - artifacts.each(&:destroy!) + def destroy_job_artifacts_with_slow_iteration(start_at) + Ci::JobArtifact.expired_before(start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index| + artifacts = relation.unlocked.with_destroy_preloads.to_a - true + parallel_destroy_batch(artifacts) if artifacts.any? + break if loop_timeout?(start_at) + break if index >= LOOP_LIMIT + end end def parallel_destroy_batch(job_artifacts) @@ -64,14 +50,14 @@ module Ci end # This is executed outside of the transaction because it depends on Redis - update_statistics_for(job_artifacts) - destroyed_artifacts_counter.increment({}, job_artifacts.size) + update_project_statistics_for(job_artifacts) + increment_monitoring_statistics(job_artifacts.size) end # This method is implemented in EE and it must do only database work def destroy_related_records_for(job_artifacts); end - def update_statistics_for(job_artifacts) + def update_project_statistics_for(job_artifacts) artifacts_by_project = job_artifacts.group_by(&:project) artifacts_by_project.each do |project, artifacts| delta = -artifacts.sum { |artifact| artifact.size.to_i } @@ -80,6 +66,11 @@ module Ci end end + def increment_monitoring_statistics(size) + destroyed_artifacts_counter.increment({}, size) + @removed_artifacts_count += size + end + def destroyed_artifacts_counter strong_memoize(:destroyed_artifacts_counter) do name = :destroyed_job_artifacts_count_total @@ -88,6 +79,10 @@ module Ci ::Gitlab::Metrics.counter(name, comment) end end + + def loop_timeout?(start_at) + Time.current > start_at + LOOP_TIMEOUT + end end end diff --git a/app/services/ci/pipelines/create_artifact_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb index bfaf317241a..9f5c445c91a 100644 --- a/app/services/ci/pipelines/create_artifact_service.rb +++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - module Pipelines - class CreateArtifactService + module PipelineArtifacts + class CoverageReportService def execute(pipeline) return unless pipeline.can_generate_coverage_reports? return if pipeline.has_coverage_reports? diff --git a/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb new file mode 100644 index 00000000000..0dbabe178da --- /dev/null +++ b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Ci + module PipelineArtifacts + class DestroyExpiredArtifactsService + include ::Gitlab::LoopHelpers + include ::Gitlab::Utils::StrongMemoize + + BATCH_SIZE = 100 + LOOP_TIMEOUT = 5.minutes + LOOP_LIMIT = 1000 + + def initialize + @removed_artifacts_count = 0 + end + + def execute + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + destroy_artifacts_batch + end + + @removed_artifacts_count + end + + private + + def destroy_artifacts_batch + artifacts = ::Ci::PipelineArtifact.expired(BATCH_SIZE).to_a + return false if artifacts.empty? + + artifacts.each(&:destroy!) + increment_stats(artifacts.size) + + true + end + + def increment_stats(size) + destroyed_artifacts_counter.increment({}, size) + @removed_artifacts_count += size + end + + def destroyed_artifacts_counter + strong_memoize(:destroyed_artifacts_counter) do + name = :destroyed_pipeline_artifacts_count_total + comment = 'Counter of destroyed expired pipeline artifacts' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end +end diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb index d9f41b7040e..a31f5e9056e 100644 --- a/app/services/ci/pipeline_trigger_service.rb +++ b/app/services/ci/pipeline_trigger_service.rb @@ -21,10 +21,10 @@ module Ci # this check is to not leak the presence of the project if user cannot read it return unless trigger.project == project - pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: params[:ref]) + pipeline = Ci::CreatePipelineService + .new(project, trigger.owner, ref: params[:ref], variables_attributes: variables) .execute(:trigger, ignore_skip_ci: true) do |pipeline| pipeline.trigger_requests.build(trigger: trigger) - pipeline.variables.build(variables) end if pipeline.persisted? @@ -44,7 +44,8 @@ module Ci # this check is to not leak the presence of the project if user cannot read it return unless can?(job.user, :read_project, project) - pipeline = Ci::CreatePipelineService.new(project, job.user, ref: params[:ref]) + pipeline = Ci::CreatePipelineService + .new(project, job.user, ref: params[:ref], variables_attributes: variables) .execute(:pipeline, ignore_skip_ci: true) do |pipeline| source = job.sourced_pipelines.build( source_pipeline: job.pipeline, @@ -53,7 +54,6 @@ module Ci project: project) pipeline.source_pipeline = source - pipeline.variables.build(variables) end if pipeline.persisted? diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb index 6adeca624a8..ebc980a9053 100644 --- a/app/services/ci/play_build_service.rb +++ b/app/services/ci/play_build_service.rb @@ -5,6 +5,10 @@ module Ci def execute(build, job_variables_attributes = nil) raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build) + if job_variables_attributes.present? && !can?(current_user, :set_pipeline_variables, project) + raise Gitlab::Access::AccessDeniedError + end + # Try to enqueue the build, otherwise create a duplicate. # if build.enqueue diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 12cdca24066..dd7b562cdb7 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -26,6 +26,27 @@ module Ci end def valid_statuses_for_build(build) + if ::Feature.enabled?(:skip_dag_manual_and_delayed_jobs, default_enabled: :yaml) + current_valid_statuses_for_build(build) + else + legacy_valid_statuses_for_build(build) + end + end + + def current_valid_statuses_for_build(build) + case build.when + when 'on_success', 'manual', 'delayed' + build.scheduling_type_dag? ? %w[success] : %w[success skipped] + when 'on_failure' + %w[failed] + when 'always' + %w[success failed skipped] + else + [] + end + end + + def legacy_valid_statuses_for_build(build) case build.when when 'on_success' build.scheduling_type_dag? ? %w[success] : %w[success skipped] diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 04d620d1d38..59691fe4ef3 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -8,8 +8,8 @@ module Ci JOB_QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze - METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'.freeze - DEFAULT_METRICS_SHARD = 'default'.freeze + METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' + DEFAULT_METRICS_SHARD = 'default' Result = Struct.new(:build, :build_json, :valid?) diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index f397ada0696..e5e79f70616 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -2,6 +2,8 @@ module Ci class RetryBuildService < ::BaseService + include Gitlab::OptimisticLocking + def self.clone_accessors %i[pipeline project ref tag options name allow_failure stage stage_id stage_idx trigger_request @@ -65,8 +67,8 @@ module Ci end def mark_subsequent_stages_as_processable(build) - build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |processable| - Gitlab::OptimisticLocking.retry_lock(processable, &:process) + build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |skipped| + retry_optimistic_lock(skipped) { |build| build.process(current_user) } end end end diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb index 45244d16393..dea4bf73a4c 100644 --- a/app/services/ci/retry_pipeline_service.rb +++ b/app/services/ci/retry_pipeline_service.rb @@ -23,7 +23,7 @@ module Ci end pipeline.builds.latest.skipped.find_each do |skipped| - retry_optimistic_lock(skipped) { |build| build.process } + retry_optimistic_lock(skipped) { |build| build.process(current_user) } end pipeline.reset_ancestor_bridges! diff --git a/app/services/ci/test_failure_history_service.rb b/app/services/ci/test_failure_history_service.rb index 99a2592ec06..61fda79a4a2 100644 --- a/app/services/ci/test_failure_history_service.rb +++ b/app/services/ci/test_failure_history_service.rb @@ -30,7 +30,6 @@ module Ci end def should_track_failures? - return false unless Feature.enabled?(:test_failure_history, project) return false unless project.default_branch_or_master == pipeline.ref # We fetch for up to MAX_TRACKABLE_FAILURES + 1 builds. So if ever we get diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb index f01d41d9414..874f4bf459a 100644 --- a/app/services/ci/update_build_state_service.rb +++ b/app/services/ci/update_build_state_service.rb @@ -111,7 +111,7 @@ module Ci Result.new(status: 200) when 'failed' - build.drop!(params[:failure_reason] || :unknown_failure) + build.drop_with_exit_code!(params[:failure_reason] || :unknown_failure, params[:exit_code]) Result.new(status: 200) else |