diff options
Diffstat (limited to 'lib/gitlab/ci/pipeline/chain')
13 files changed, 95 insertions, 311 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 9c12d46cede..07a3aff1862 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -11,9 +11,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def perform! + ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project) + return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables| cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| auto_cancel_interruptible_pipelines(cancelables_batch.ids) end @@ -27,13 +29,19 @@ module Gitlab private - def auto_cancelable_pipelines - project.all_pipelines.created_after(1.week.ago) + def auto_cancelable_pipelines(ff_enabled) + relation = project.all_pipelines + .created_after(1.week.ago) .ci_and_parent_sources .for_ref(pipeline.ref) - .id_not_in(pipeline.same_family_pipeline_ids) .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled + + if ff_enabled + relation.id_not_in(pipeline.id) + else + relation.id_not_in(pipeline.same_family_pipeline_ids) + end end def auto_cancel_interruptible_pipelines(pipeline_ids) @@ -41,6 +49,14 @@ module Gitlab .id_in(pipeline_ids) .with_only_interruptible_builds .each do |cancelable_pipeline| + Gitlab::AppLogger.info( + class: self.class.name, + message: "Pipeline #{pipeline.id} auto-canceling pipeline #{cancelable_pipeline.id}", + canceled_pipeline_id: cancelable_pipeline.id, + canceled_by_pipeline_id: pipeline.id, + canceled_by_pipeline_source: pipeline.source + ) + # cascade_to_children not needed because we iterate through descendants here cancelable_pipeline.cancel_running( auto_canceled_by_pipeline_id: pipeline.id, diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 14c320f77bf..76d4a05bf30 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -121,11 +121,7 @@ module Gitlab end def observe_jobs_count_in_alive_pipelines - jobs_count = if Feature.enabled?(:ci_limit_active_jobs_early, project) - project.all_pipelines.jobs_count_in_alive_pipelines - else - project.all_pipelines.builds_count_in_alive_pipelines - end + jobs_count = project.all_pipelines.jobs_count_in_alive_pipelines metrics.active_jobs_histogram .observe({ plan: project.actual_plan_name }, jobs_count) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a14dec48619..d41213ef6dd 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -9,15 +9,6 @@ module Gitlab include Chain::Helpers include ::Gitlab::Utils::StrongMemoize - SOURCES = [ - Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter, - Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge, - Gitlab::Ci::Pipeline::Chain::Config::Content::Repository, - Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject, - Gitlab::Ci::Pipeline::Chain::Config::Content::Remote, - Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops - ].freeze - def perform! if pipeline_config&.exists? @pipeline.build_pipeline_config(content: pipeline_config.content) @@ -36,8 +27,6 @@ module Gitlab def pipeline_config strong_memoize(:pipeline_config) do - next legacy_find_config if ::Feature.disabled?(:ci_project_pipeline_config_refactoring, project) - ::Gitlab::Ci::ProjectConfig.new( project: project, sha: @pipeline.sha, custom_content: @command.content, @@ -45,24 +34,9 @@ module Gitlab ) end end - - def legacy_find_config - sources.each do |source| - config = source.new(@pipeline, @command) - return config if config.exists? - end - - nil - end - - def sources - SOURCES - end end end end end end end - -Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb deleted file mode 100644 index 4947e2eb879..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class AutoDevops < Source - def content - strong_memoize(:content) do - next unless project&.auto_devops_enabled? - - template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name) - YAML.dump('include' => [{ 'template' => template.full_name }]) - end - end - - def source - :auto_devops_source - end - - private - - def template_name - 'Auto-DevOps' - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb b/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb deleted file mode 100644 index 39ffa2d4e25..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Bridge < Source - def content - return unless @command.bridge - - @command.bridge.yaml_for_downstream - end - - def source - :bridge_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb deleted file mode 100644 index 092e7d43371..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class ExternalProject < Source - def content - strong_memoize(:content) do - next unless external_project_path? - - path_file, path_project, ref = extract_location_tokens - - config_location = { 'project' => path_project, 'file' => path_file } - config_location['ref'] = ref if ref.present? - - YAML.dump('include' => [config_location]) - end - end - - def source - :external_project_source - end - - private - - # Example: path/to/.gitlab-ci.yml@another-group/another-project - def external_project_path? - ci_config_path =~ /\A.+(yml|yaml)@.+\z/ - end - - # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname - def extract_location_tokens - path_file, path_project = ci_config_path.split('@', 2) - - if path_project.include? ":" - project, ref = path_project.split(':', 2) - [path_file, project, ref] - else - [path_file, path_project] - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb deleted file mode 100644 index 9954aedc4b7..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Parameter < Source - UnsupportedSourceError = Class.new(StandardError) - - def content - strong_memoize(:content) do - next unless command.content.present? - - command.content - end - end - - def source - :parameter_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb deleted file mode 100644 index 4990a5a6eb5..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Remote < Source - def content - strong_memoize(:content) do - next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) - - YAML.dump('include' => [{ 'remote' => ci_config_path }]) - end - end - - def source - :remote_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb deleted file mode 100644 index 0752b099d3d..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Repository < Source - def content - strong_memoize(:content) do - next unless file_in_repository? - - YAML.dump('include' => [{ 'local' => ci_config_path }]) - end - end - - def source - :repository_source - end - - private - - def file_in_repository? - return unless project - return unless @pipeline.sha - - project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present? - rescue GRPC::NotFound, GRPC::Internal - nil - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb deleted file mode 100644 index 69dca1568b6..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/source.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - # When removing ci_project_pipeline_config_refactoring, this and its subclasses will be removed. - class Source - include Gitlab::Utils::StrongMemoize - - DEFAULT_YAML_FILE = '.gitlab-ci.yml' - - attr_reader :command - - def initialize(pipeline, command) - @pipeline = pipeline - @command = command - end - - def exists? - strong_memoize(:exists) do - content.present? - end - end - - def content - raise NotImplementedError - end - - def source - raise NotImplementedError - end - - def project - @project ||= @pipeline.project - end - - def ci_config_path - @ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb new file mode 100644 index 00000000000..8b26416edf7 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class ActiveJobs < Chain::Base + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Ci::Pipeline::Chain::Helpers + + LIMIT_NAME = :ci_active_jobs + MESSAGE = "Project exceeded the allowed number of jobs in active pipelines. Retry later." + + def perform! + return unless limits.exceeded?(LIMIT_NAME, count_jobs_in_alive_pipelines) + + error(MESSAGE, drop_reason: :job_activity_limit_exceeded) + + Gitlab::AppLogger.info( + class: self.class.name, + message: MESSAGE, + project_id: project.id, + plan: project.actual_plan_name) + end + + def break? + pipeline.errors.any? + end + + private + + def namespace + strong_memoize(:namespace) do + project.namespace + end + end + + def limits + strong_memoize(:limits) do + namespace.actual_limits + end + end + + def count_jobs_in_alive_pipelines + strong_memoize(:count_jobs_in_alive_pipelines) do + count_persisted_jobs_in_all_alive_pipelines + count_current_pipeline_jobs + end + end + + def count_current_pipeline_jobs + command.pipeline_seed.size + end + + def count_persisted_jobs_in_all_alive_pipelines + project.all_pipelines.jobs_count_in_alive_pipelines + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb deleted file mode 100644 index 3706dd0b9f6..00000000000 --- a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Limit - class JobActivity < Chain::Base - def perform! - # to be overridden in EE - end - - def break? - false # to be overridden in EE - end - end - end - end - end - end -end - -Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 654e24be8e1..4bec8355732 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -25,6 +25,8 @@ module Gitlab return error('Failed to build the pipeline!') end + set_pipeline_name + raise Populate::PopulateError if pipeline.persisted? end @@ -34,6 +36,15 @@ module Gitlab private + def set_pipeline_name + return if Feature.disabled?(:pipeline_name, pipeline.project) || + @command.yaml_processor_result.workflow_name.blank? + + name = @command.yaml_processor_result.workflow_name + + pipeline.build_pipeline_metadata(project: pipeline.project, title: name) + end + def stage_names # We filter out `.pre/.post` stages, as they alone are not considered # a complete pipeline: |