From f6a3028be7c20cd509685258ba7e50d9d6a68114 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 12 Aug 2020 15:10:02 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- lib/gitlab/ci/features.rb | 8 +++++ .../ci/pipeline/chain/cancel_pending_pipelines.rb | 40 ++++++++++++++++++++++ lib/gitlab/ci/pipeline/chain/command.rb | 4 ++- .../ci/pipeline/chain/config/content/parameter.rb | 1 - lib/gitlab/ci/pipeline/chain/helpers.rb | 8 +++-- lib/gitlab/ci/pipeline/chain/metrics.rb | 23 +++++++++++++ lib/gitlab/ci/pipeline/chain/pipeline/process.rb | 24 +++++++++++++ lib/gitlab/ci/pipeline/chain/sequence.rb | 19 +++------- lib/gitlab/ci/pipeline/chain/stop_dry_run.rb | 22 ++++++++++++ lib/gitlab/ci/pipeline/metrics.rb | 18 ++++++++++ lib/gitlab/json.rb | 28 +++++++++++++++ lib/gitlab/static_site_editor/config.rb | 4 ++- 12 files changed, 180 insertions(+), 19 deletions(-) create mode 100644 lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb create mode 100644 lib/gitlab/ci/pipeline/chain/metrics.rb create mode 100644 lib/gitlab/ci/pipeline/chain/pipeline/process.rb create mode 100644 lib/gitlab/ci/pipeline/chain/stop_dry_run.rb (limited to 'lib') diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 356e881a0a5..d4b3c58ec56 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -76,9 +76,17 @@ module Gitlab ::Feature.enabled?(:ci_job_entry_matches_all_keys) end + def self.lint_creates_pipeline_with_dry_run?(project) + ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project) + end + def self.reset_ci_minutes_for_all_namespaces? ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false) end + + def self.expand_names_for_cross_pipeline_artifacts?(project) + ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb new file mode 100644 index 00000000000..468f3bc4689 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CancelPendingPipelines < Chain::Base + include Chain::Helpers + + def perform! + return unless project.auto_cancel_pending_pipelines? + + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables| + cancelables.find_each do |cancelable| + cancelable.auto_cancel_running(pipeline) + end + end + end + + def break? + false + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def auto_cancelable_pipelines + project.ci_pipelines + .where(ref: pipeline.ref) + .where.not(id: pipeline.same_family_pipeline_ids) + .where.not(sha: project.commit(pipeline.ref).try(:id)) + .alive_or_scheduled + .with_only_interruptible_builds + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 74b28b181bc..dbaa6951e64 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: :config_content, :config_processor, :stage_seeds ) do @@ -22,6 +22,8 @@ module Gitlab end end + alias_method :dry_run?, :dry_run + def branch_exists? strong_memoize(:is_branch) do project.repository.branch_exists?(ref) diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb index 3dd216b33d1..9954aedc4b7 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb @@ -12,7 +12,6 @@ module Gitlab def content strong_memoize(:content) do next unless command.content.present? - raise UnsupportedSourceError, "#{command.source} not a dangling build" unless command.dangling_build? command.content end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index aba7dab508d..d7271df1694 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -6,13 +6,13 @@ module Gitlab module Chain module Helpers def error(message, config_error: false, drop_reason: nil) - if config_error && command.save_incompleted + if config_error drop_reason = :config_error pipeline.yaml_errors = message end pipeline.add_error_message(message) - pipeline.drop!(drop_reason) if drop_reason + pipeline.drop!(drop_reason) if drop_reason && persist_pipeline? # TODO: consider not to rely on AR errors directly as they can be # polluted with other unrelated errors (e.g. state machine) @@ -23,6 +23,10 @@ module Gitlab def warning(message) pipeline.add_warning_message(message) end + + def persist_pipeline? + command.save_incompleted && !pipeline.readonly? + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb new file mode 100644 index 00000000000..0d7449813b4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Metrics < Chain::Base + def perform! + counter.increment(source: @pipeline.source) + end + + def break? + false + end + + def counter + ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb new file mode 100644 index 00000000000..98e9e1479bc --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Pipeline + # After pipeline has been successfully created we can start processing it. + class Process < Chain::Base + def perform! + ::Ci::ProcessPipelineService + .new(@pipeline) + .execute(nil, initial_process: true) + end + + def break? + false + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 204c7725214..dc648568129 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,21 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @completed = [] @start = Time.now end def build! - @sequence.each do |chain| - step = chain.new(@pipeline, @command) + @sequence.each do |step_class| + step = step_class.new(@pipeline, @command) step.perform! break if step.break? - - @completed.push(step) end - @pipeline.tap do - yield @pipeline, self if block_given? - - @command.observe_creation_duration(Time.now - @start) - @command.observe_pipeline_size(@pipeline) - end - end + @command.observe_creation_duration(Time.now - @start) + @command.observe_pipeline_size(@pipeline) - def complete? - @completed.size == @sequence.size + @pipeline end end end diff --git a/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb new file mode 100644 index 00000000000..0e9add4ee74 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + # During the dry run we don't want to persist the pipeline and skip + # all the other steps that operate on a persisted context. + # This causes the chain to break at this point. + class StopDryRun < Chain::Base + def perform! + # no-op + end + + def break? + @command.dry_run? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 649da745eea..ef6981eeb4d 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -36,6 +36,24 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def pipelines_created_counter + strong_memoize(:pipelines_created_count) do + name = :pipelines_created_total + comment = 'Counter of pipelines created' + + Gitlab::Metrics.counter(name, comment) + end + end + + def pipelines_created_counter + strong_memoize(:pipelines_created_count) do + name = :pipelines_created_total + comment = 'Counter of pipelines created' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 21f837c58bb..d6681347f42 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -220,5 +220,33 @@ module Gitlab end end end + + class LimitedEncoder + LimitExceeded = Class.new(StandardError) + + # Generates JSON for an object or raise an error if the resulting json string is too big + # + # @param object [Hash, Array, Object] must be hash, array, or an object that responds to .to_h or .to_json + # @param limit [Integer] max size of the resulting json string + # @return [String] + # @raise [LimitExceeded] if the resulting json string is bigger than the specified limit + def self.encode(object, limit: 25.megabytes) + return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder) + + buffer = [] + buffer_size = 0 + + ::Yajl::Encoder.encode(object) do |data_chunk| + chunk_size = data_chunk.bytesize + + raise LimitExceeded if buffer_size + chunk_size > limit + + buffer << data_chunk + buffer_size += chunk_size + end + + buffer.join('') + end + end end end diff --git a/lib/gitlab/static_site_editor/config.rb b/lib/gitlab/static_site_editor/config.rb index 08ed6599a6e..d335a434335 100644 --- a/lib/gitlab/static_site_editor/config.rb +++ b/lib/gitlab/static_site_editor/config.rb @@ -3,7 +3,7 @@ module Gitlab module StaticSiteEditor class Config - SUPPORTED_EXTENSIONS = %w[.md .md.erb].freeze + SUPPORTED_EXTENSIONS = %w[.md].freeze def initialize(repository, ref, file_path, return_url) @repository = repository @@ -42,6 +42,8 @@ module Gitlab end def extension_supported? + return true if file_path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) + SUPPORTED_EXTENSIONS.any? { |ext| file_path.end_with?(ext) } end -- cgit v1.2.1