diff options
Diffstat (limited to 'lib/gitlab/ci')
55 files changed, 489 insertions, 755 deletions
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 1fac00337a3..97988d8aa13 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -30,6 +30,8 @@ module Gitlab Converter.new.convert(ansi, state) end + Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride + class Converter def on_0(_) reset @@ -278,9 +280,7 @@ module Gitlab close_open_tags - # TODO: replace OpenStruct with a better type - # https://gitlab.com/gitlab-org/gitlab/issues/34305 - OpenStruct.new( + Ansi2html::Result.new( html: @out.force_encoding(Encoding.default_external), state: state, append: append, diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb index 4fcb5168847..375e6b4a96f 100644 --- a/lib/gitlab/ci/build/cache.rb +++ b/lib/gitlab/ci/build/cache.rb @@ -7,39 +7,22 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize def initialize(cache, pipeline) - if multiple_cache_per_job? - cache = Array.wrap(cache) - @cache = cache.map do |cache| - Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) - end - else - @cache = Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) + cache = Array.wrap(cache) + @cache = cache.map do |cache| + Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache) end end def cache_attributes strong_memoize(:cache_attributes) do - if multiple_cache_per_job? - if @cache.empty? - {} - else - { options: { cache: @cache.map(&:attributes) } } - end + if @cache.empty? + {} else - @cache.build_attributes + { options: { cache: @cache.map(&:attributes) } } end end end - - private - - def multiple_cache_per_job? - strong_memoize(:multiple_cache_per_job) do - ::Gitlab::Ci::Features.multiple_cache_per_job? - end - end end end end diff --git a/lib/gitlab/ci/build/releaser.rb b/lib/gitlab/ci/build/releaser.rb index facb5f619bd..9720bb1123a 100644 --- a/lib/gitlab/ci/build/releaser.rb +++ b/lib/gitlab/ci/build/releaser.rb @@ -18,8 +18,9 @@ module Gitlab command = BASE_COMMAND.dup single_flags.each { |k, v| command.concat(" --#{k.to_s.dasherize} \"#{v}\"") } array_commands.each { |k, v| v.each { |elem| command.concat(" --#{k.to_s.singularize.dasherize} \"#{elem}\"") } } + asset_links.each { |link| command.concat(" --assets-link #{stringified_json(link)}") } - [command] + [command.freeze] end private @@ -31,6 +32,14 @@ module Gitlab def array_commands config.slice(*ARRAY_FLAGS) end + + def asset_links + config.dig(:assets, :links) || [] + end + + def stringified_json(object) + "#{object.to_json.to_json}" + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 23b0c93a3ee..9c6428d701c 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,13 +17,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :ref + attr_reader :root, :context, :ref, :source - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil, source: nil) @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) @context.set_deadline(TIMEOUT_SECONDS) @ref = ref + @source = source @config = expand_config(config) @@ -128,4 +129,4 @@ module Gitlab end end -Gitlab::Ci::Config.prepend_if_ee('EE::Gitlab::Ci::ConfigEE') +Gitlab::Ci::Config.prepend_mod_with('Gitlab::Ci::ConfigEE') diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index f9688c500d2..ab79add688b 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -4,88 +4,52 @@ module Gitlab module Ci class Config module Entry - ## - # Entry that represents a cache configuration - # - class Cache < ::Gitlab::Config::Entry::Simplifiable - strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) } - strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) } - - class Caches < ::Gitlab::Config::Entry::ComposableArray - include ::Gitlab::Config::Entry::Validatable - - MULTIPLE_CACHE_LIMIT = 4 - - validations do - validate do - unless config.is_a?(Hash) || config.is_a?(Array) - errors.add(:config, 'can only be a Hash or an Array') - end - - if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT - errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") - end - end - end - - def initialize(*args) - super - - @key = nil - end - - def composable_class - Entry::Cache::Cache + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze + DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } end end - class Cache < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[key untracked paths when policy].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze - DEFAULT_POLICY = 'pull-push' - ALLOWED_WHEN = %w[on_success on_failure always].freeze - DEFAULT_WHEN = 'on_success' + entry :key, Entry::Key, + description: 'Cache key used to define a cache affinity.' - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true - - with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } - end - end + entry :untracked, ::Gitlab::Config::Entry::Boolean, + description: 'Cache all untracked files.' - entry :key, Entry::Key, - description: 'Cache key used to define a cache affinity.' + entry :paths, Entry::Paths, + description: 'Specify which paths should be cached across builds.' - entry :untracked, ::Gitlab::Config::Entry::Boolean, - description: 'Cache all untracked files.' + attributes :policy, :when - entry :paths, Entry::Paths, - description: 'Specify which paths should be cached across builds.' + def value + result = super - attributes :policy, :when + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN - def value - result = super - - result[:key] = key_value - result[:policy] = policy || DEFAULT_POLICY - # Use self.when to avoid conflict with reserved word - result[:when] = self.when || DEFAULT_WHEN - - result - end + result end class UnknownStrategy < ::Gitlab::Config::Entry::Node diff --git a/lib/gitlab/ci/config/entry/caches.rb b/lib/gitlab/ci/config/entry/caches.rb new file mode 100644 index 00000000000..75240599c9c --- /dev/null +++ b/lib/gitlab/ci/config/entry/caches.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents caches configuration + # + class Caches < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + MULTIPLE_CACHE_LIMIT = 4 + + validations do + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + + if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT + errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") + end + end + end + + def initialize(*args) + super + + @key = nil + end + + def composable_class + Entry::Cache + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index ab493ff7d78..eaaf9f69102 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -37,7 +37,7 @@ module Gitlab description: 'Script that will be executed after each job.', inherit: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', inherit: true diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a20b802be58..c8e8f0bc1fc 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -64,7 +64,7 @@ module Gitlab description: 'Commands that will be executed when finishing job.', inherit: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Cache definition for this job.', inherit: true @@ -200,4 +200,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Job.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Job') +::Gitlab::Ci::Config::Entry::Job.prepend_mod_with('Gitlab::Ci::Config::Entry::Job') diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index b3cf0f9e0fd..29dc48c7b42 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -118,4 +118,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Need.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Need') +::Gitlab::Ci::Config::Entry::Need.prepend_mod_with('Gitlab::Ci::Config::Entry::Need') diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index dd01cfeedff..11b202ddde9 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -56,4 +56,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Needs.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Needs') +::Gitlab::Ci::Config::Entry::Needs.prepend_mod_with('Gitlab::Ci::Config::Entry::Needs') diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 54ef84b965a..e6290ef2479 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -61,7 +61,7 @@ module Gitlab description: 'Deprecated: stages for this pipeline.', reserved: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', reserved: true diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 12e182b38fc..c8e4d9ed763 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -18,9 +18,8 @@ module Gitlab Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end - # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/224199 - def self.store_pipeline_messages?(project) - ::Feature.enabled?(:ci_store_pipeline_messages, project, default_enabled: true) + def self.merge_base_pipeline_for_metrics_comparison?(project) + Feature.enabled?(:merge_base_pipeline_for_metrics_comparison, project, default_enabled: :yaml) end def self.raise_job_rules_without_workflow_rules_warning? @@ -47,22 +46,17 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.validate_build_dependencies?(project) - ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && - ::Feature.disabled?(:ci_validate_build_dependencies_override, project) - end - def self.display_quality_on_mr_diff?(project) - ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) - end - - def self.multiple_cache_per_job? - ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) + ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: :yaml) end def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end + + def self.background_pipeline_retry_endpoint?(project) + ::Feature.enabled?(:background_pipeline_retry_endpoint, project) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index a6ae249fa58..0b94debb24e 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -123,4 +123,4 @@ module Gitlab end end -Gitlab::Ci::Jwt.prepend_if_ee('::EE::Gitlab::Ci::Jwt') +Gitlab::Ci::Jwt.prepend_mod_with('Gitlab::Ci::Jwt') diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 2baa8faf849..3469537a2e2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -15,8 +15,8 @@ module Gitlab } end - def self.fabricate!(file_type, *args) - parsers.fetch(file_type.to_sym).new(*args) + def self.fabricate!(file_type, *args, **kwargs) + parsers.fetch(file_type.to_sym).new(*args, **kwargs) rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end @@ -28,4 +28,4 @@ module Gitlab end end -Gitlab::Ci::Parsers.prepend_if_ee('::EE::Gitlab::Ci::Parsers') +Gitlab::Ci::Parsers.prepend_mod_with('Gitlab::Ci::Parsers') diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index eb3adf713d4..d6b3af674a6 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -121,7 +121,7 @@ module Gitlab # Using `Integer()` here to raise exception on invalid values [Integer(line["number"]), Integer(line["hits"])] end - rescue + rescue StandardError raise InvalidLineInformationError, "Line information had invalid values" end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index abfbe18e23f..f9afa58f915 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -19,7 +19,7 @@ module Gitlab end rescue JSON::ParserError terraform_reports.add_plan(job_id, invalid_tfplan(:invalid_json_format, job_details)) - rescue + rescue StandardError details = job_details || {} plan_name = job_id || 'failed_tf_plan' terraform_reports.add_plan(plan_name, invalid_tfplan(:unknown_error, details)) diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 50cd703da4a..ca7fbde6713 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -31,7 +31,7 @@ module Gitlab def ensure_test_cases_limited!(total_parsed, limit) return unless limit > 0 && total_parsed > limit - raise JunitParserError.new("number of test cases exceeded the limit of #{limit}") + raise JunitParserError, "number of test cases exceeded the limit of #{limit}" end def all_cases(root, parent = nil, &blk) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a7680f6e593..3c150ca26bb 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -52,4 +52,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Config::Content') +Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 8f1c49563f2..49ec1250a5f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -16,6 +16,7 @@ module Gitlab project: project, ref: @pipeline.ref, sha: @pipeline.sha, + source: @pipeline.source, user: current_user, parent_pipeline: parent_pipeline } @@ -31,7 +32,7 @@ module Gitlab @pipeline.merged_yaml = result.merged_yaml - rescue => ex + rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, sha: @pipeline.sha diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 9988b6f18ed..09158bf8bfd 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -19,6 +19,8 @@ module Gitlab # polluted with other unrelated errors (e.g. state machine) # https://gitlab.com/gitlab-org/gitlab/-/issues/220823 pipeline.errors.add(:base, message) + + pipeline.errors.full_messages end def warning(message) diff --git a/lib/gitlab/ci/pipeline/chain/limit/activity.rb b/lib/gitlab/ci/pipeline/chain/limit/activity.rb index 3c64278e305..ef9235477db 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/activity.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/activity.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::Activity') +Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Activity') diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb index 2e8b437252f..3706dd0b9f6 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') +Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') diff --git a/lib/gitlab/ci/pipeline/chain/limit/size.rb b/lib/gitlab/ci/pipeline/chain/limit/size.rb index 739648840e9..761bdb1c484 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/size.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/size.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::Size.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::Size') +Gitlab::Ci::Pipeline::Chain::Limit::Size.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Size') diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index df92e229f12..e4e4f4f484a 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -11,7 +11,14 @@ module Gitlab def perform! if skipped? - @pipeline.skip if @command.save_incompleted + if @command.save_incompleted + # Project iid must be called outside a transaction, so we ensure it is set here + # otherwise it may be set within the state transition transaction of the skip call + # which it will lock the InternalId row for the whole transaction + @pipeline.ensure_project_iid! + + @pipeline.skip + end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 55c125e03d5..1c1f7abb6f6 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -57,4 +57,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Validate::Abilities.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::Abilities') +Gitlab::Ci::Pipeline::Chain::Validate::Abilities.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::Abilities') diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 6149d2f04d7..539b44513f0 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -54,7 +54,7 @@ module Gitlab else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" end - rescue => ex + rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) true @@ -147,4 +147,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External') +Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::External') diff --git a/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb b/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb new file mode 100644 index 00000000000..e3588aa3027 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class SecurityOrchestrationPolicy < Chain::Base + include Chain::Helpers + + def perform! + # no-op + end + + def break? + false + end + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy') diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 6cb6fd3920d..84b88374a7f 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -13,6 +13,13 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_security_orchestration_policy_processing_duration_histogram + name = :gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds + comment = 'Pipeline security orchestration policy processing duration' + + ::Gitlab::Metrics.histogram(name, comment) + end + def self.pipeline_size_histogram name = :gitlab_ci_pipeline_size_builds comment = 'Pipeline size' @@ -56,6 +63,21 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end + + def ci_minutes_exceeded_builds_counter + name = :ci_minutes_exceeded_builds_counter + comment = 'Count of builds dropped due to CI minutes exceeded' + + Gitlab::Metrics.counter(name, comment) + end + + def self.gitlab_ci_difference_live_vs_actual_minutes + name = :gitlab_ci_difference_live_vs_actual_minutes + comment = 'Comparison between CI minutes consumption from live tracking vs actual consumption' + labels = {} + buckets = [-120.0, -60.0, -30.0, -10.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0] + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7ecb9a1db16..46e4373ec85 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -10,7 +10,7 @@ module Gitlab QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze - QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 15, 20, 30, 60].freeze METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' DEFAULT_METRICS_SHARD = 'default' diff --git a/lib/gitlab/ci/reports/codequality_mr_diff.rb b/lib/gitlab/ci/reports/codequality_mr_diff.rb index e60a075e3f5..0595b6f966a 100644 --- a/lib/gitlab/ci/reports/codequality_mr_diff.rb +++ b/lib/gitlab/ci/reports/codequality_mr_diff.rb @@ -6,8 +6,8 @@ module Gitlab class CodequalityMrDiff attr_reader :files - def initialize(raw_report) - @raw_report = raw_report + def initialize(new_errors) + @new_errors = new_errors @files = {} build_report! end @@ -15,7 +15,7 @@ module Gitlab private def build_report! - codequality_files = @raw_report.all_degradations.each_with_object({}) do |degradation, codequality_files| + codequality_files = @new_errors.each_with_object({}) do |degradation, codequality_files| unless codequality_files[degradation.dig(:location, :path)].present? codequality_files[degradation.dig(:location, :path)] = [] end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index 37d0da38065..c110dbf98be 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -13,7 +13,7 @@ module Gitlab def load! recent_failures_count.each do |key_hash, count| - failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master) + failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_main) end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 787dee3b267..cbd72f54ff4 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -20,6 +20,7 @@ module Gitlab scheduler_failure: 'scheduler failure', data_integrity_failure: 'data integrity failure', forward_deployment_failure: 'forward deployment failure', + pipeline_loop_detected: 'job would create infinitely looping pipelines', invalid_bridge_trigger: 'downstream pipeline trigger definition is invalid', downstream_bridge_project_not_found: 'downstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', @@ -28,7 +29,8 @@ module Gitlab secrets_provider_not_found: 'secrets provider can not be found', reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', project_deleted: 'pipeline project was deleted', - user_blocked: 'pipeline user was blocked' + user_blocked: 'pipeline user was blocked', + ci_quota_exceeded: 'no more CI minutes available' }.freeze private_constant :REASONS @@ -68,4 +70,4 @@ module Gitlab end end -Gitlab::Ci::Status::Build::Failed.prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') +Gitlab::Ci::Status::Build::Failed.prepend_mod_with('Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 4779c8d3d53..e7ed2081f6a 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -11,6 +11,8 @@ module Gitlab attr_reader :subject, :user + delegate :cache_key, to: :subject + def initialize(subject, user) @subject = subject @user = user diff --git a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml deleted file mode 100644 index 7182b96594d..00000000000 --- a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml +++ /dev/null @@ -1,52 +0,0 @@ -# -# You can use artifacts to pass data to jobs in later stages. -# For more information, see https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job might build an important file, and pass it to later jobs." - - echo "This is the content of the important file" > important-file.txt - artifacts: - paths: - - important-file.txt - -test-job-with-artifacts: - stage: test - script: - - echo "This job uses the artifact from the job in the earlier stage." - - cat important-file.txt - - echo "It creates another file, and adds it to the artifacts." - - echo "This is a second important file" > important-file2.txt - artifacts: - paths: - - important-file2.txt - -test-job-with-no-artifacts: - stage: test - dependencies: [] # Use to skip downloading any artifacts - script: - - echo "This job does not get the artifacts from other jobs." - - cat important-file.txt || exit 0 - -deploy-job-with-all-artifacts: - stage: deploy - script: - - echo "By default, jobs download all available artifacts." - - cat important-file.txt - - cat important-file2.txt - -deploy-job-with-1-artifact: - stage: deploy - dependencies: - - build-job # Download artifacts from only this job - script: - - echo "You can configure a job to download artifacts from only certain jobs." - - cat important-file.txt - - cat important-file2.txt || exit 0 diff --git a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml deleted file mode 100644 index 382bac09ed7..00000000000 --- a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml +++ /dev/null @@ -1,36 +0,0 @@ -# -# You can define common tasks and run them before or after the main scripts in jobs. -# For more information, see: -# - https://docs.gitlab.com/ee/ci/yaml/README.html#before_script -# - https://docs.gitlab.com/ee/ci/yaml/README.html#after_script -# - -stages: - - test - -default: - before_script: - - echo "This script runs before the main script in every job, unless the job overrides it." - - echo "It may set up common dependencies, for example." - after_script: - - echo "This script runs after the main script in every job, unless the job overrides it." - - echo "It may do some common final clean up tasks" - -job-standard: - stage: test - script: - - echo "This job uses both of the globally defined before and after scripts." - -job-override-before: - stage: test - before_script: - - echo "Use a different before_script in this job." - script: - - echo "This job uses its own before_script, and the global after_script." - -job-override-after: - stage: test - after_script: - - echo "Use a different after_script in this job." - script: - - echo "This job uses its own after_script, and the global before_script." diff --git a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml deleted file mode 100644 index 5f27def74c9..00000000000 --- a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml +++ /dev/null @@ -1,53 +0,0 @@ -# -# A manual job is a type of job that is not executed automatically and must be explicitly started by a user. -# To make a job manual, add when: manual to its configuration. -# For more information, see https://docs.gitlab.com/ee/ci/yaml/README.html#whenmanual -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job is not a manual job" - -manual-build: - stage: build - script: - - echo "This manual job passes after you trigger it." - when: manual - -manual-build-allowed-to-fail: - stage: build - script: - - echo "This manual job fails after you trigger it." - - echo "It is allowed to fail, so the pipeline does not fail. - when: manual - allow_failure: true # Default behavior - -test-job: - stage: test - script: - - echo "This is a normal test job" - - echo "It runs when the when the build stage completes." - - echo "It does not need to wait for the manual jobs in the build stage to run." - -manual-test-not-allowed-to-fail: - stage: test - script: - - echo "This manual job fails after you trigger it." - - echo "It is NOT allowed to fail, so the pipeline is marked as failed - - echo "when this job completes." - - exit 1 - when: manual - allow_failure: false # Optional behavior - -deploy-job: - stage: deploy - script: - - echo "This is a normal deploy job" - - echo "If a manual job that isn't allowed to fail ran in an earlier stage and failed, - - echo "this job does not run". diff --git a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml deleted file mode 100644 index aced628aacb..00000000000 --- a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# A pipeline is composed of independent jobs that run scripts, grouped into stages. -# Stages run in sequential order, but jobs within stages run in parallel. -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job runs in the build stage, which runs first." - -test-job1: - stage: test - script: - - echo "This job runs in the test stage." - - echo "It only starts when the job in the build stage completes successfully." - -test-job2: - stage: test - script: - - echo "This job also runs in the test stage." - - echo "This job can run at the same time as test-job2." - -deploy-job: - stage: deploy - script: - - echo "This job runs in the deploy stage." - - echo "It only runs when both jobs in the test stage complete successfully" diff --git a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml deleted file mode 100644 index 2b8cf7bab44..00000000000 --- a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Variables can be used to for more dynamic behavior in jobs and scripts. -# For more information, see https://docs.gitlab.com/ee/ci/variables/README.html -# - -stages: - - test - -variables: - VAR1: "Variable 1 defined globally" - -use-a-variable: - stage: test - script: - - echo "You can use variables in jobs." - - echo "The content of 'VAR1' is = $VAR1" - -override-a-variable: - stage: test - variables: - VAR1: "Variable 1 was overriden in in the job." - script: - - echo "You can override global variables in jobs." - - echo "The content of 'VAR1' is = $VAR1" - -define-a-new-variable: - stage: test - variables: - VAR2: "Variable 2 is new and defined in the job only." - script: - - echo "You can mix global variables with variables defined in jobs." - - echo "The content of 'VAR1' is = $VAR1" - - echo "The content of 'VAR2' is = $VAR2" - -incorrect-variable-usage: - stage: test - script: - - echo "You can't use variables only defined in other jobs." - - echo "The content of 'VAR2' is = $VAR2" - -predefined-variables: - stage: test - script: - - echo "Some variables are predefined by GitLab CI/CD, for example:" - - echo "The commit author's username is $GITLAB_USER_LOGIN" - - echo "The commit branch is $CI_COMMIT_BRANCH" - - echo "The project path is $CI_PROJECT_PATH" diff --git a/lib/gitlab/ci/templates/Getting-started.yml b/lib/gitlab/ci/templates/Getting-started.yml new file mode 100644 index 00000000000..4dc88418671 --- /dev/null +++ b/lib/gitlab/ci/templates/Getting-started.yml @@ -0,0 +1,39 @@ +# This is a sample GitLab CI/CD configuration file that should run without any modifications. +# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, +# it uses echo commands to simulate the pipeline execution. +# +# A pipeline is composed of independent jobs that run scripts, grouped into stages. +# Stages run in sequential order, but jobs within stages run in parallel. +# +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages + +stages: # List of stages for jobs, and their order of execution + - build + - test + - deploy + +build-job: # This job runs in the build stage, which runs first. + stage: build + script: + - echo "Compiling the code..." + - echo "Compile complete. + +unit-test-job: # This job runs in the test stage. + stage: test # It only starts when the job in the build stage completes successfully. + script: + - echo "Running unit tests... This will take about 60 seconds." + - sleep 60 + - echo "Code coverage is 90%" + +lint-test-job: # This job also runs in the test stage. + stage: test # It can run at the same time as unit-test-job (in parallel). + script: + - echo "Linting code... This will take about 10 seconds." + - sleep 10 + - echo "No lint issues found." + +deploy-job: # This job runs in the deploy stage. + stage: deploy # It only runs when *both* jobs in the test stage complete successfully. + script: + - echo "Deploying application..." + - echo "Application successfully deployed." diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml index c7fb1321055..7f33d048c1e 100644 --- a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml @@ -29,12 +29,8 @@ default: before_script: - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content -stages: - - init_and_plan - - cloudrail - init_and_plan: - stage: init_and_plan + stage: build image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 rules: - if: $SAST_DISABLED @@ -52,7 +48,7 @@ init_and_plan: - ./**/.terraform cloudrail_scan: - stage: cloudrail + stage: test image: indeni/cloudrail-cli:1.2.44 rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..5216a46745c --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -0,0 +1,77 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +browser_performance: + stage: performance + image: docker:19.03.12 + allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" + SITESPEED_IMAGE: sitespeedio/sitespeed.io + SITESPEED_VERSION: 14.1.0 + SITESPEED_OPTIONS: '' + services: + - docker:19.03.12-dind + script: + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js + - mkdir sitespeed-results + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + else + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + fi + - mv sitespeed-results/data/performance.json browser-performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + browser_performance: browser-performance.json + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$PERFORMANCE_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 1c25d9d583b..abcb347b146 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,11 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - name: "docker:20.10.6-dind" + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then @@ -16,6 +17,8 @@ build: fi - /build/build.sh rules: + - if: '$BUILD_DISABLED' + when: never - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -26,4 +29,6 @@ build_artifact: - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n" - exit 1 rules: + - if: '$BUILD_DISABLED' + when: never - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 3faf07546de..45bddb1bc6a 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -46,27 +46,23 @@ review: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN on_stop: stop-review - only: - - branches - except: - - master + rules: + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH stop-review: <<: *deploy stage: cleanup script: - oc delete all -l "app=$APP" - when: manual variables: APP: review-$CI_COMMIT_REF_NAME GIT_STRATEGY: none environment: name: review/$CI_COMMIT_REF_NAME action: stop - only: - - branches - except: - - master + rules: + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + when: manual staging: <<: *deploy @@ -77,8 +73,8 @@ staging: environment: name: staging url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH production: <<: *deploy @@ -86,9 +82,9 @@ production: variables: APP: production APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN - when: manual environment: name: production url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index bf42cd52605..90fad1550ff 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -11,6 +11,7 @@ stages: - fuzz variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" FUZZAPI_PROFILE: Quick FUZZAPI_VERSION: "1.6" FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml @@ -24,7 +25,7 @@ variables: # available (non 500 response to HTTP(s)) FUZZAPI_SERVICE_START_TIMEOUT: "300" # - FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} # apifuzzer_fuzz_unlicensed: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 215029dc952..8fa33026011 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -5,266 +5,30 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: latest - FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml - FUZZAPI_TIMEOUT: 30 - FUZZAPI_REPORT: gl-api-fuzzing-report.json - FUZZAPI_REPORT_ASSET_PATH: assets - # - FUZZAPI_D_NETWORK: testing-net - # - # Wait up to 5 minutes for API Fuzzer and target url to become - # available (non 500 response to HTTP(s)) - FUZZAPI_SERVICE_START_TIMEOUT: "300" - # - FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine - # - -apifuzzer_fuzz_unlicensed: - stage: fuzz - allow_failure: true - rules: - - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' - - when: never - script: - - | - echo "Error: Your GitLab project is not licensed for API Fuzzing." - - exit 1 + FUZZAPI_VERSION: "1" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} apifuzzer_fuzz: stage: fuzz - image: - name: $FUZZAPI_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:80 - FUZZAPI_NEW_REPORT: 1 - FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log - TZ: America/Los_Angeles - allow_failure: true - rules: - - if: $FUZZAPI_D_TARGET_IMAGE - when: never - - if: $FUZZAPI_D_WORKER_IMAGE - when: never - - if: $API_FUZZING_DISABLED - when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - script: - # - # Validate options - - | - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT - - $FUZZAPI_LOG_SCANNER - reports: - api_fuzzing: $FUZZAPI_REPORT - -apifuzzer_fuzz_dnd: - stage: fuzz - image: docker:19.03.12 - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + image: $FUZZAPI_IMAGE allow_failure: true rules: - - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null - when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - services: - - docker:19.03.12-dind + - if: $CI_COMMIT_BRANCH script: - # - # - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # - - docker network create --driver bridge $FUZZAPI_D_NETWORK - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start peach testing engine container - - | - docker run -d \ - --name apifuzzer \ - --network $FUZZAPI_D_NETWORK \ - -e Proxy:Port=8000 \ - -e TZ=America/Los_Angeles \ - -e GITLAB_FEATURES \ - -p 80:80 \ - -p 8000:8000 \ - -p 514:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - dotnet /peach/Peach.Web.dll - # - # Start target container - - | - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ - docker run -d \ - --name target \ - --network $FUZZAPI_D_NETWORK \ - $FUZZAPI_D_TARGET_ENV \ - $FUZZAPI_D_TARGET_PORTS \ - $FUZZAPI_D_TARGET_VOLUME \ - --restart=no \ - $FUZZAPI_D_TARGET_IMAGE \ - ; fi - # - # Start worker container if provided - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ - $FUZZAPI_D_WORKER_ENV \ - $FUZZAPI_D_WORKER_PORTS \ - $FUZZAPI_D_WORKER_VOLUME \ - --restart=no \ - $FUZZAPI_D_WORKER_IMAGE \ - ; fi - # - # Start API Fuzzing provided worker if no other worker present - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -v $CI_PROJECT_DIR:/app \ - -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ - -p 81:80 \ - -p 8001:8000 \ - -p 515:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - worker-entry \ - ; fi - # - # Propagate exit code from api fuzzing scanner (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - after_script: - # - # Shutdown all containers - - echo "Stopping all containers" - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - docker stop worker - - docker stop apifuzzer - # - # Save docker logs - - docker logs apifuzzer &> gl-api_fuzzing-logs.log - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - docker logs worker &> gl-api_fuzzing-worker-logs.log - # + - /peach/analyzer-fuzz-api artifacts: when: always paths: - - ./gl-api_fuzzing*.log - - ./gl-api_fuzzing*.zip - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT + - gl-assets + - gl-api-fuzzing-report.json + - gl-*.log reports: - api_fuzzing: $FUZZAPI_REPORT + api_fuzzing: gl-api-fuzzing-report.json # end diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml new file mode 100644 index 00000000000..b40c4e982f7 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -0,0 +1,48 @@ +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST-API.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API: +# +# stages: +# - build +# - test +# - deploy +# - dast + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html + +# Configure the scanning tool with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html) +# List of variables available to configure the DAST API scanning tool: +# https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables + +variables: + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_VERSION: "1" + DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + +dast_api: + stage: dast + image: $DAST_API_IMAGE + allow_failure: true + rules: + - if: $DAST_API_DISABLED + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 533f8bb25f8..b6282da18a4 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -22,19 +22,6 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - # - DAST_API_PROFILE: Full - DAST_API_VERSION: latest - DAST_API_CONFIG: .gitlab-dast-api.yml - DAST_API_TIMEOUT: 30 - DAST_API_REPORT: gl-dast-api-report.json - DAST_API_REPORT_ASSET_PATH: assets - # - # Wait up to 5 minutes for API Security and target url to become - # available (non 500 response to HTTP(s)) - DAST_API_SERVICE_START_TIMEOUT: "300" - # - DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine dast: stage: dast @@ -51,11 +38,6 @@ dast: reports: dast: gl-dast-report.json rules: - - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION || - $DAST_API_OPENAPI || - $DAST_API_POSTMAN_COLLECTION || - $DAST_API_HAR ) - when: never - if: $DAST_DISABLED when: never - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && @@ -71,72 +53,4 @@ dast: - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE - if: $CI_COMMIT_BRANCH && - $DAST_API_BETA == null && $DAST_API_SPECIFICATION - -dast_api: - stage: dast - image: - name: $DAST_API_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - API_SECURITY_MODE: DAST - DAST_API_NEW_REPORT: 1 - DAST_API_PROJECT: $CI_PROJECT_PATH - DAST_API_API: http://127.0.0.1:5000 - DAST_API_LOG_SCANNER: gl-dast-api-scanner.log - TZ: America/Los_Angeles - allow_failure: true - rules: - - if: $DAST_API_BETA == null - when: never - - if: $DAST_DISABLED - when: never - - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED && - $DAST_API_SPECIFICATION == null && - $DAST_API_OPENAPI == null && - $DAST_API_POSTMAN_COLLECTION == null && - $DAST_API_HAR == null - when: never - - if: $DAST_API_SPECIFICATION == null && - $DAST_API_OPENAPI == null && - $DAST_API_POSTMAN_COLLECTION == null && - $DAST_API_HAR == null - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdast\b/ - script: - # - # Run user provided pre-script - - sh -c "$DAST_API_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $DAST_API_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$DAST_API_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $DAST_API_REPORT_ASSET_PATH - - $DAST_API_REPORT - - $DAST_API_LOG_SCANNER - - gl-*.log - reports: - dast: $DAST_API_REPORT diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 3039d64514b..53d68c24d26 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -8,8 +8,8 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" + DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 2 @@ -45,6 +45,8 @@ gemnasium-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ @@ -71,6 +73,8 @@ gemnasium-maven-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ @@ -92,6 +96,8 @@ gemnasium-python-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ @@ -120,6 +126,8 @@ bundler-audit-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ @@ -138,6 +146,8 @@ retire-js-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /retire.js/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /retire.js/ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 3ebccfbba4a..a8d45e80356 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -155,13 +155,8 @@ gosec-sast: exists: - '**/*.go' -mobsf-android-sast: +.mobsf-sast: extends: .sast-analyzer - services: - # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile - # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 - alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: @@ -169,7 +164,9 @@ mobsf-android-sast: # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - MOBSF_API_KEY: key + +mobsf-android-sast: + extends: .mobsf-sast rules: - if: $SAST_DISABLED when: never @@ -179,23 +176,11 @@ mobsf-android-sast: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: + - '**/*.apk' - '**/AndroidManifest.xml' mobsf-ios-sast: - extends: .sast-analyzer - services: - # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile - # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 - alias: mobsf - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - MOBSF_API_KEY: key + extends: .mobsf-sast rules: - if: $SAST_DISABLED when: never @@ -205,6 +190,7 @@ mobsf-ios-sast: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: + - '**/*.ipa' - '**/*.xcodeproj/*' nodejs-scan-sast: @@ -292,15 +278,14 @@ semgrep-sast: # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /semgrep/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' + $SAST_DEFAULT_ANALYZERS =~ /semgrep/ exists: - '**/*.py' - '**/*.js' diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 232c320562b..ac975fbbeab 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -13,11 +13,11 @@ variables: SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, klar, clair-vulnerabilities-db, license-finder, - dast + dast, api-fuzzing SECURE_BINARIES_DOWNLOAD_IMAGES: "true" SECURE_BINARIES_PUSH_IMAGES: "true" @@ -134,6 +134,13 @@ secrets: variables: SECURE_BINARIES_ANALYZER_VERSION: "3" +semgrep: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsemgrep\b/ + sobelow: extends: .download_images only: @@ -241,3 +248,12 @@ dast: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ + +api-fuzzing: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "1" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bapi-fuzzing\b/ diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 7e2828d010f..6b9db1c2e0f 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -56,6 +56,6 @@ apply: - terraform apply -input=false $PLAN dependencies: - plan - when: manual - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f0621165f8a --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -0,0 +1,52 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +stages: + - build + - test + - deploy + - performance + +browser_performance: + stage: performance + image: docker:git + variables: + URL: '' + SITESPEED_IMAGE: sitespeedio/sitespeed.io + SITESPEED_VERSION: 14.1.0 + SITESPEED_OPTIONS: '' + services: + - docker:stable-dind + script: + - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js + - mkdir sitespeed-results + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - mv sitespeed-results/data/performance.json browser-performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + browser_performance: browser-performance.json diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c25c4339c35..c4757edf74e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -317,4 +317,4 @@ module Gitlab end end -::Gitlab::Ci::Trace.prepend_if_ee('EE::Gitlab::Ci::Trace') +::Gitlab::Ci::Trace.prepend_mod_with('Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 618438c8887..fdc598c025a 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -93,7 +93,7 @@ module Gitlab end nil - rescue + rescue StandardError # if bad regex or something goes wrong we dont want to interrupt transition # so we just silently ignore error for now end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index dc4951f76bb..a8c1002f2b9 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -141,7 +141,7 @@ module Gitlab end def error!(message) - raise ValidationError.new(message) + raise ValidationError, message end end end |