summaryrefslogtreecommitdiff
path: root/lib/gitlab/ci/pipeline/chain
diff options
context:
space:
mode:
Diffstat (limited to 'lib/gitlab/ci/pipeline/chain')
-rw-r--r--lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb24
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb6
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content.rb26
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb34
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/bridge.rb25
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/external_project.rb51
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/parameter.rb29
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/remote.rb27
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/repository.rb38
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/source.rb49
-rw-r--r--lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb63
-rw-r--r--lib/gitlab/ci/pipeline/chain/limit/job_activity.rb23
-rw-r--r--lib/gitlab/ci/pipeline/chain/populate.rb11
13 files changed, 95 insertions, 311 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
index 9c12d46cede..07a3aff1862 100644
--- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
+++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
@@ -11,9 +11,11 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def perform!
+ ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project)
+ return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline
return unless project.auto_cancel_pending_pipelines?
- Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables|
+ Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables|
cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch|
auto_cancel_interruptible_pipelines(cancelables_batch.ids)
end
@@ -27,13 +29,19 @@ module Gitlab
private
- def auto_cancelable_pipelines
- project.all_pipelines.created_after(1.week.ago)
+ def auto_cancelable_pipelines(ff_enabled)
+ relation = project.all_pipelines
+ .created_after(1.week.ago)
.ci_and_parent_sources
.for_ref(pipeline.ref)
- .id_not_in(pipeline.same_family_pipeline_ids)
.where_not_sha(project.commit(pipeline.ref).try(:id))
.alive_or_scheduled
+
+ if ff_enabled
+ relation.id_not_in(pipeline.id)
+ else
+ relation.id_not_in(pipeline.same_family_pipeline_ids)
+ end
end
def auto_cancel_interruptible_pipelines(pipeline_ids)
@@ -41,6 +49,14 @@ module Gitlab
.id_in(pipeline_ids)
.with_only_interruptible_builds
.each do |cancelable_pipeline|
+ Gitlab::AppLogger.info(
+ class: self.class.name,
+ message: "Pipeline #{pipeline.id} auto-canceling pipeline #{cancelable_pipeline.id}",
+ canceled_pipeline_id: cancelable_pipeline.id,
+ canceled_by_pipeline_id: pipeline.id,
+ canceled_by_pipeline_source: pipeline.source
+ )
+
# cascade_to_children not needed because we iterate through descendants here
cancelable_pipeline.cancel_running(
auto_canceled_by_pipeline_id: pipeline.id,
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 14c320f77bf..76d4a05bf30 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -121,11 +121,7 @@ module Gitlab
end
def observe_jobs_count_in_alive_pipelines
- jobs_count = if Feature.enabled?(:ci_limit_active_jobs_early, project)
- project.all_pipelines.jobs_count_in_alive_pipelines
- else
- project.all_pipelines.builds_count_in_alive_pipelines
- end
+ jobs_count = project.all_pipelines.jobs_count_in_alive_pipelines
metrics.active_jobs_histogram
.observe({ plan: project.actual_plan_name }, jobs_count)
diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb
index a14dec48619..d41213ef6dd 100644
--- a/lib/gitlab/ci/pipeline/chain/config/content.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/content.rb
@@ -9,15 +9,6 @@ module Gitlab
include Chain::Helpers
include ::Gitlab::Utils::StrongMemoize
- SOURCES = [
- Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter,
- Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge,
- Gitlab::Ci::Pipeline::Chain::Config::Content::Repository,
- Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject,
- Gitlab::Ci::Pipeline::Chain::Config::Content::Remote,
- Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops
- ].freeze
-
def perform!
if pipeline_config&.exists?
@pipeline.build_pipeline_config(content: pipeline_config.content)
@@ -36,8 +27,6 @@ module Gitlab
def pipeline_config
strong_memoize(:pipeline_config) do
- next legacy_find_config if ::Feature.disabled?(:ci_project_pipeline_config_refactoring, project)
-
::Gitlab::Ci::ProjectConfig.new(
project: project, sha: @pipeline.sha,
custom_content: @command.content,
@@ -45,24 +34,9 @@ module Gitlab
)
end
end
-
- def legacy_find_config
- sources.each do |source|
- config = source.new(@pipeline, @command)
- return config if config.exists?
- end
-
- nil
- end
-
- def sources
- SOURCES
- end
end
end
end
end
end
end
-
-Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Config::Content')
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb
deleted file mode 100644
index 4947e2eb879..00000000000
--- a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Config
- class Content
- class AutoDevops < Source
- def content
- strong_memoize(:content) do
- next unless project&.auto_devops_enabled?
-
- template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name)
- YAML.dump('include' => [{ 'template' => template.full_name }])
- end
- end
-
- def source
- :auto_devops_source
- end
-
- private
-
- def template_name
- 'Auto-DevOps'
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb b/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb
deleted file mode 100644
index 39ffa2d4e25..00000000000
--- a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Config
- class Content
- class Bridge < Source
- def content
- return unless @command.bridge
-
- @command.bridge.yaml_for_downstream
- end
-
- def source
- :bridge_source
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb
deleted file mode 100644
index 092e7d43371..00000000000
--- a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Config
- class Content
- class ExternalProject < Source
- def content
- strong_memoize(:content) do
- next unless external_project_path?
-
- path_file, path_project, ref = extract_location_tokens
-
- config_location = { 'project' => path_project, 'file' => path_file }
- config_location['ref'] = ref if ref.present?
-
- YAML.dump('include' => [config_location])
- end
- end
-
- def source
- :external_project_source
- end
-
- private
-
- # Example: path/to/.gitlab-ci.yml@another-group/another-project
- def external_project_path?
- ci_config_path =~ /\A.+(yml|yaml)@.+\z/
- end
-
- # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname
- def extract_location_tokens
- path_file, path_project = ci_config_path.split('@', 2)
-
- if path_project.include? ":"
- project, ref = path_project.split(':', 2)
- [path_file, project, ref]
- else
- [path_file, path_project]
- end
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
deleted file mode 100644
index 9954aedc4b7..00000000000
--- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Config
- class Content
- class Parameter < Source
- UnsupportedSourceError = Class.new(StandardError)
-
- def content
- strong_memoize(:content) do
- next unless command.content.present?
-
- command.content
- end
- end
-
- def source
- :parameter_source
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb
deleted file mode 100644
index 4990a5a6eb5..00000000000
--- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Config
- class Content
- class Remote < Source
- def content
- strong_memoize(:content) do
- next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https])
-
- YAML.dump('include' => [{ 'remote' => ci_config_path }])
- end
- end
-
- def source
- :remote_source
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb
deleted file mode 100644
index 0752b099d3d..00000000000
--- a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Config
- class Content
- class Repository < Source
- def content
- strong_memoize(:content) do
- next unless file_in_repository?
-
- YAML.dump('include' => [{ 'local' => ci_config_path }])
- end
- end
-
- def source
- :repository_source
- end
-
- private
-
- def file_in_repository?
- return unless project
- return unless @pipeline.sha
-
- project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present?
- rescue GRPC::NotFound, GRPC::Internal
- nil
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb
deleted file mode 100644
index 69dca1568b6..00000000000
--- a/lib/gitlab/ci/pipeline/chain/config/content/source.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Config
- class Content
- # When removing ci_project_pipeline_config_refactoring, this and its subclasses will be removed.
- class Source
- include Gitlab::Utils::StrongMemoize
-
- DEFAULT_YAML_FILE = '.gitlab-ci.yml'
-
- attr_reader :command
-
- def initialize(pipeline, command)
- @pipeline = pipeline
- @command = command
- end
-
- def exists?
- strong_memoize(:exists) do
- content.present?
- end
- end
-
- def content
- raise NotImplementedError
- end
-
- def source
- raise NotImplementedError
- end
-
- def project
- @project ||= @pipeline.project
- end
-
- def ci_config_path
- @ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE
- end
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb
new file mode 100644
index 00000000000..8b26416edf7
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Chain
+ module Limit
+ class ActiveJobs < Chain::Base
+ include ::Gitlab::Utils::StrongMemoize
+ include ::Gitlab::Ci::Pipeline::Chain::Helpers
+
+ LIMIT_NAME = :ci_active_jobs
+ MESSAGE = "Project exceeded the allowed number of jobs in active pipelines. Retry later."
+
+ def perform!
+ return unless limits.exceeded?(LIMIT_NAME, count_jobs_in_alive_pipelines)
+
+ error(MESSAGE, drop_reason: :job_activity_limit_exceeded)
+
+ Gitlab::AppLogger.info(
+ class: self.class.name,
+ message: MESSAGE,
+ project_id: project.id,
+ plan: project.actual_plan_name)
+ end
+
+ def break?
+ pipeline.errors.any?
+ end
+
+ private
+
+ def namespace
+ strong_memoize(:namespace) do
+ project.namespace
+ end
+ end
+
+ def limits
+ strong_memoize(:limits) do
+ namespace.actual_limits
+ end
+ end
+
+ def count_jobs_in_alive_pipelines
+ strong_memoize(:count_jobs_in_alive_pipelines) do
+ count_persisted_jobs_in_all_alive_pipelines + count_current_pipeline_jobs
+ end
+ end
+
+ def count_current_pipeline_jobs
+ command.pipeline_seed.size
+ end
+
+ def count_persisted_jobs_in_all_alive_pipelines
+ project.all_pipelines.jobs_count_in_alive_pipelines
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb
deleted file mode 100644
index 3706dd0b9f6..00000000000
--- a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Pipeline
- module Chain
- module Limit
- class JobActivity < Chain::Base
- def perform!
- # to be overridden in EE
- end
-
- def break?
- false # to be overridden in EE
- end
- end
- end
- end
- end
- end
-end
-
-Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::JobActivity')
diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb
index 654e24be8e1..4bec8355732 100644
--- a/lib/gitlab/ci/pipeline/chain/populate.rb
+++ b/lib/gitlab/ci/pipeline/chain/populate.rb
@@ -25,6 +25,8 @@ module Gitlab
return error('Failed to build the pipeline!')
end
+ set_pipeline_name
+
raise Populate::PopulateError if pipeline.persisted?
end
@@ -34,6 +36,15 @@ module Gitlab
private
+ def set_pipeline_name
+ return if Feature.disabled?(:pipeline_name, pipeline.project) ||
+ @command.yaml_processor_result.workflow_name.blank?
+
+ name = @command.yaml_processor_result.workflow_name
+
+ pipeline.build_pipeline_metadata(project: pipeline.project, title: name)
+ end
+
def stage_names
# We filter out `.pre/.post` stages, as they alone are not considered
# a complete pipeline: