summaryrefslogtreecommitdiff
path: root/app/services/ci
diff options
context:
space:
mode:
Diffstat (limited to 'app/services/ci')
-rw-r--r--app/services/ci/create_downstream_pipeline_service.rb4
-rw-r--r--app/services/ci/create_pipeline_service.rb16
-rw-r--r--app/services/ci/create_web_ide_terminal_service.rb2
-rw-r--r--app/services/ci/destroy_expired_job_artifacts_service.rb57
-rw-r--r--app/services/ci/pipeline_artifacts/coverage_report_service.rb (renamed from app/services/ci/pipelines/create_artifact_service.rb)4
-rw-r--r--app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb52
-rw-r--r--app/services/ci/pipeline_trigger_service.rb8
-rw-r--r--app/services/ci/play_build_service.rb4
-rw-r--r--app/services/ci/process_build_service.rb21
-rw-r--r--app/services/ci/register_job_service.rb4
-rw-r--r--app/services/ci/retry_build_service.rb6
-rw-r--r--app/services/ci/retry_pipeline_service.rb2
-rw-r--r--app/services/ci/test_failure_history_service.rb1
-rw-r--r--app/services/ci/update_build_state_service.rb2
14 files changed, 134 insertions, 49 deletions
diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb
index 86d0cf079fc..629d85b041f 100644
--- a/app/services/ci/create_downstream_pipeline_service.rb
+++ b/app/services/ci/create_downstream_pipeline_service.rb
@@ -33,9 +33,7 @@ module Ci
pipeline_params.fetch(:target_revision))
downstream_pipeline = service.execute(
- pipeline_params.fetch(:source), **pipeline_params[:execute_params]) do |pipeline|
- pipeline.variables.build(@bridge.downstream_variables)
- end
+ pipeline_params.fetch(:source), **pipeline_params[:execute_params])
downstream_pipeline.tap do |pipeline|
update_bridge_status!(@bridge, pipeline)
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index dbe81521cfc..d3001e54288 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -27,6 +27,7 @@ module Ci
Gitlab::Ci::Pipeline::Chain::Limit::JobActivity,
Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines,
Gitlab::Ci::Pipeline::Chain::Metrics,
+ Gitlab::Ci::Pipeline::Chain::TemplateUsage,
Gitlab::Ci::Pipeline::Chain::Pipeline::Process].freeze
# Create a new pipeline in the specified project.
@@ -81,7 +82,11 @@ module Ci
.new(pipeline, command, SEQUENCE)
.build!
- schedule_head_pipeline_update if pipeline.persisted?
+ if pipeline.persisted?
+ schedule_head_pipeline_update
+ record_conversion_event
+ create_namespace_onboarding_action
+ end
# If pipeline is not persisted, try to recover IID
pipeline.reset_project_iid unless pipeline.persisted?
@@ -116,6 +121,15 @@ module Ci
end
end
+ def record_conversion_event
+ Experiments::RecordConversionEventWorker.perform_async(:ci_syntax_templates, current_user.id)
+ Experiments::RecordConversionEventWorker.perform_async(:pipelines_empty_state, current_user.id)
+ end
+
+ def create_namespace_onboarding_action
+ Namespaces::OnboardingPipelineCreatedWorker.perform_async(project.namespace_id)
+ end
+
def extra_options(content: nil, dry_run: false)
{ content: content, dry_run: dry_run }
end
diff --git a/app/services/ci/create_web_ide_terminal_service.rb b/app/services/ci/create_web_ide_terminal_service.rb
index a78281aed16..785d82094b9 100644
--- a/app/services/ci/create_web_ide_terminal_service.rb
+++ b/app/services/ci/create_web_ide_terminal_service.rb
@@ -6,7 +6,7 @@ module Ci
TerminalCreationError = Class.new(StandardError)
- TERMINAL_NAME = 'terminal'.freeze
+ TERMINAL_NAME = 'terminal'
attr_reader :terminal
diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb
index 6e7caba8545..7d8a3c17abe 100644
--- a/app/services/ci/destroy_expired_job_artifacts_service.rb
+++ b/app/services/ci/destroy_expired_job_artifacts_service.rb
@@ -12,6 +12,10 @@ module Ci
EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock'
LOCK_TIMEOUT = 6.minutes
+ def initialize
+ @removed_artifacts_count = 0
+ end
+
##
# Destroy expired job artifacts on GitLab instance
#
@@ -20,40 +24,22 @@ module Ci
# which is scheduled every 7 minutes.
def execute
in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
- loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
- destroy_artifacts_batch
- end
+ destroy_job_artifacts_with_slow_iteration(Time.current)
end
- end
-
- private
- def destroy_artifacts_batch
- destroy_job_artifacts_batch || destroy_pipeline_artifacts_batch
+ @removed_artifacts_count
end
- def destroy_job_artifacts_batch
- artifacts = Ci::JobArtifact
- .expired(BATCH_SIZE)
- .unlocked
- .with_destroy_preloads
- .to_a
-
- return false if artifacts.empty?
-
- parallel_destroy_batch(artifacts)
- true
- end
-
- # TODO: Make sure this can also be parallelized
- # https://gitlab.com/gitlab-org/gitlab/-/issues/270973
- def destroy_pipeline_artifacts_batch
- artifacts = Ci::PipelineArtifact.expired(BATCH_SIZE).to_a
- return false if artifacts.empty?
+ private
- artifacts.each(&:destroy!)
+ def destroy_job_artifacts_with_slow_iteration(start_at)
+ Ci::JobArtifact.expired_before(start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index|
+ artifacts = relation.unlocked.with_destroy_preloads.to_a
- true
+ parallel_destroy_batch(artifacts) if artifacts.any?
+ break if loop_timeout?(start_at)
+ break if index >= LOOP_LIMIT
+ end
end
def parallel_destroy_batch(job_artifacts)
@@ -64,14 +50,14 @@ module Ci
end
# This is executed outside of the transaction because it depends on Redis
- update_statistics_for(job_artifacts)
- destroyed_artifacts_counter.increment({}, job_artifacts.size)
+ update_project_statistics_for(job_artifacts)
+ increment_monitoring_statistics(job_artifacts.size)
end
# This method is implemented in EE and it must do only database work
def destroy_related_records_for(job_artifacts); end
- def update_statistics_for(job_artifacts)
+ def update_project_statistics_for(job_artifacts)
artifacts_by_project = job_artifacts.group_by(&:project)
artifacts_by_project.each do |project, artifacts|
delta = -artifacts.sum { |artifact| artifact.size.to_i }
@@ -80,6 +66,11 @@ module Ci
end
end
+ def increment_monitoring_statistics(size)
+ destroyed_artifacts_counter.increment({}, size)
+ @removed_artifacts_count += size
+ end
+
def destroyed_artifacts_counter
strong_memoize(:destroyed_artifacts_counter) do
name = :destroyed_job_artifacts_count_total
@@ -88,6 +79,10 @@ module Ci
::Gitlab::Metrics.counter(name, comment)
end
end
+
+ def loop_timeout?(start_at)
+ Time.current > start_at + LOOP_TIMEOUT
+ end
end
end
diff --git a/app/services/ci/pipelines/create_artifact_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
index bfaf317241a..9f5c445c91a 100644
--- a/app/services/ci/pipelines/create_artifact_service.rb
+++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Ci
- module Pipelines
- class CreateArtifactService
+ module PipelineArtifacts
+ class CoverageReportService
def execute(pipeline)
return unless pipeline.can_generate_coverage_reports?
return if pipeline.has_coverage_reports?
diff --git a/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb
new file mode 100644
index 00000000000..0dbabe178da
--- /dev/null
+++ b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Ci
+ module PipelineArtifacts
+ class DestroyExpiredArtifactsService
+ include ::Gitlab::LoopHelpers
+ include ::Gitlab::Utils::StrongMemoize
+
+ BATCH_SIZE = 100
+ LOOP_TIMEOUT = 5.minutes
+ LOOP_LIMIT = 1000
+
+ def initialize
+ @removed_artifacts_count = 0
+ end
+
+ def execute
+ loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
+ destroy_artifacts_batch
+ end
+
+ @removed_artifacts_count
+ end
+
+ private
+
+ def destroy_artifacts_batch
+ artifacts = ::Ci::PipelineArtifact.expired(BATCH_SIZE).to_a
+ return false if artifacts.empty?
+
+ artifacts.each(&:destroy!)
+ increment_stats(artifacts.size)
+
+ true
+ end
+
+ def increment_stats(size)
+ destroyed_artifacts_counter.increment({}, size)
+ @removed_artifacts_count += size
+ end
+
+ def destroyed_artifacts_counter
+ strong_memoize(:destroyed_artifacts_counter) do
+ name = :destroyed_pipeline_artifacts_count_total
+ comment = 'Counter of destroyed expired pipeline artifacts'
+
+ ::Gitlab::Metrics.counter(name, comment)
+ end
+ end
+ end
+ end
+end
diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb
index d9f41b7040e..a31f5e9056e 100644
--- a/app/services/ci/pipeline_trigger_service.rb
+++ b/app/services/ci/pipeline_trigger_service.rb
@@ -21,10 +21,10 @@ module Ci
# this check is to not leak the presence of the project if user cannot read it
return unless trigger.project == project
- pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: params[:ref])
+ pipeline = Ci::CreatePipelineService
+ .new(project, trigger.owner, ref: params[:ref], variables_attributes: variables)
.execute(:trigger, ignore_skip_ci: true) do |pipeline|
pipeline.trigger_requests.build(trigger: trigger)
- pipeline.variables.build(variables)
end
if pipeline.persisted?
@@ -44,7 +44,8 @@ module Ci
# this check is to not leak the presence of the project if user cannot read it
return unless can?(job.user, :read_project, project)
- pipeline = Ci::CreatePipelineService.new(project, job.user, ref: params[:ref])
+ pipeline = Ci::CreatePipelineService
+ .new(project, job.user, ref: params[:ref], variables_attributes: variables)
.execute(:pipeline, ignore_skip_ci: true) do |pipeline|
source = job.sourced_pipelines.build(
source_pipeline: job.pipeline,
@@ -53,7 +54,6 @@ module Ci
project: project)
pipeline.source_pipeline = source
- pipeline.variables.build(variables)
end
if pipeline.persisted?
diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb
index 6adeca624a8..ebc980a9053 100644
--- a/app/services/ci/play_build_service.rb
+++ b/app/services/ci/play_build_service.rb
@@ -5,6 +5,10 @@ module Ci
def execute(build, job_variables_attributes = nil)
raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build)
+ if job_variables_attributes.present? && !can?(current_user, :set_pipeline_variables, project)
+ raise Gitlab::Access::AccessDeniedError
+ end
+
# Try to enqueue the build, otherwise create a duplicate.
#
if build.enqueue
diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb
index 12cdca24066..dd7b562cdb7 100644
--- a/app/services/ci/process_build_service.rb
+++ b/app/services/ci/process_build_service.rb
@@ -26,6 +26,27 @@ module Ci
end
def valid_statuses_for_build(build)
+ if ::Feature.enabled?(:skip_dag_manual_and_delayed_jobs, default_enabled: :yaml)
+ current_valid_statuses_for_build(build)
+ else
+ legacy_valid_statuses_for_build(build)
+ end
+ end
+
+ def current_valid_statuses_for_build(build)
+ case build.when
+ when 'on_success', 'manual', 'delayed'
+ build.scheduling_type_dag? ? %w[success] : %w[success skipped]
+ when 'on_failure'
+ %w[failed]
+ when 'always'
+ %w[success failed skipped]
+ else
+ []
+ end
+ end
+
+ def legacy_valid_statuses_for_build(build)
case build.when
when 'on_success'
build.scheduling_type_dag? ? %w[success] : %w[success skipped]
diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb
index 04d620d1d38..59691fe4ef3 100644
--- a/app/services/ci/register_job_service.rb
+++ b/app/services/ci/register_job_service.rb
@@ -8,8 +8,8 @@ module Ci
JOB_QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze
- METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'.freeze
- DEFAULT_METRICS_SHARD = 'default'.freeze
+ METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
+ DEFAULT_METRICS_SHARD = 'default'
Result = Struct.new(:build, :build_json, :valid?)
diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb
index f397ada0696..e5e79f70616 100644
--- a/app/services/ci/retry_build_service.rb
+++ b/app/services/ci/retry_build_service.rb
@@ -2,6 +2,8 @@
module Ci
class RetryBuildService < ::BaseService
+ include Gitlab::OptimisticLocking
+
def self.clone_accessors
%i[pipeline project ref tag options name
allow_failure stage stage_id stage_idx trigger_request
@@ -65,8 +67,8 @@ module Ci
end
def mark_subsequent_stages_as_processable(build)
- build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |processable|
- Gitlab::OptimisticLocking.retry_lock(processable, &:process)
+ build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |skipped|
+ retry_optimistic_lock(skipped) { |build| build.process(current_user) }
end
end
end
diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb
index 45244d16393..dea4bf73a4c 100644
--- a/app/services/ci/retry_pipeline_service.rb
+++ b/app/services/ci/retry_pipeline_service.rb
@@ -23,7 +23,7 @@ module Ci
end
pipeline.builds.latest.skipped.find_each do |skipped|
- retry_optimistic_lock(skipped) { |build| build.process }
+ retry_optimistic_lock(skipped) { |build| build.process(current_user) }
end
pipeline.reset_ancestor_bridges!
diff --git a/app/services/ci/test_failure_history_service.rb b/app/services/ci/test_failure_history_service.rb
index 99a2592ec06..61fda79a4a2 100644
--- a/app/services/ci/test_failure_history_service.rb
+++ b/app/services/ci/test_failure_history_service.rb
@@ -30,7 +30,6 @@ module Ci
end
def should_track_failures?
- return false unless Feature.enabled?(:test_failure_history, project)
return false unless project.default_branch_or_master == pipeline.ref
# We fetch for up to MAX_TRACKABLE_FAILURES + 1 builds. So if ever we get
diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb
index f01d41d9414..874f4bf459a 100644
--- a/app/services/ci/update_build_state_service.rb
+++ b/app/services/ci/update_build_state_service.rb
@@ -111,7 +111,7 @@ module Ci
Result.new(status: 200)
when 'failed'
- build.drop!(params[:failure_reason] || :unknown_failure)
+ build.drop_with_exit_code!(params[:failure_reason] || :unknown_failure, params[:exit_code])
Result.new(status: 200)
else