diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
| commit | 311b0269b4eb9839fa63f80c8d7a58f32b8138a0 (patch) | |
| tree | 07e7870bca8aed6d61fdcc810731c50d2c40af47 /lib/gitlab/ci | |
| parent | 27909cef6c4170ed9205afa7426b8d3de47cbb0c (diff) | |
| download | gitlab-ce-311b0269b4eb9839fa63f80c8d7a58f32b8138a0.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-5-stable-eev14.5.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
55 files changed, 482 insertions, 212 deletions
diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index 3cfed8e5e2c..b0fad026ec5 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -45,14 +45,6 @@ module Gitlab end def read_zip_file!(file_path) - if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: :yaml) - read_with_new_artifact_file_reader(file_path) - else - read_with_legacy_artifact_file_reader(file_path) - end - end - - def read_with_new_artifact_file_reader(file_path) job.artifacts_file.use_open_file do |file| zip_file = Zip::File.new(file, false, true) entry = zip_file.find_entry(file_path) @@ -69,25 +61,6 @@ module Gitlab end end - def read_with_legacy_artifact_file_reader(file_path) - job.artifacts_file.use_file do |archive_path| - Zip::File.open(archive_path) do |zip_file| - entry = zip_file.find_entry(file_path) - unless entry - raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!" - end - - if entry.name_is_directory? - raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!" - end - - zip_file.get_input_stream(entry) do |is| - is.read - end - end - end - end - def max_archive_size_in_mb ActiveSupport::NumberHelper.number_to_human_size(MAX_ARCHIVE_SIZE) end diff --git a/lib/gitlab/ci/artifacts/metrics.rb b/lib/gitlab/ci/artifacts/metrics.rb index 656f4d2cc13..03459c4bf36 100644 --- a/lib/gitlab/ci/artifacts/metrics.rb +++ b/lib/gitlab/ci/artifacts/metrics.rb @@ -6,10 +6,14 @@ module Gitlab class Metrics include Gitlab::Utils::StrongMemoize - def increment_destroyed_artifacts(size) + def increment_destroyed_artifacts_count(size) destroyed_artifacts_counter.increment({}, size.to_i) end + def increment_destroyed_artifacts_bytes(bytes) + destroyed_artifacts_bytes_counter.increment({}, bytes) + end + private def destroyed_artifacts_counter @@ -20,6 +24,15 @@ module Gitlab ::Gitlab::Metrics.counter(name, comment) end end + + def destroyed_artifacts_bytes_counter + strong_memoize(:destroyed_artifacts_bytes_counter) do + name = :destroyed_job_artifacts_bytes_total + comment = 'Counter of bytes of destroyed expired job artifacts' + + ::Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index 6ab567dff7c..4950a7616c8 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -10,7 +10,9 @@ class Gitlab::Ci::Build::AutoRetry RETRY_OVERRIDES = { ci_quota_exceeded: 0, no_matching_runner: 0, - missing_dependency_failure: 0 + missing_dependency_failure: 0, + forward_deployment_failure: 0, + environment_creation_failure: 0 }.freeze def initialize(build) diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb index 02b97ea76e9..c7ea7c78e2f 100644 --- a/lib/gitlab/ci/build/context/base.rb +++ b/lib/gitlab/ci/build/context/base.rb @@ -5,6 +5,8 @@ module Gitlab module Build module Context class Base + include Gitlab::Utils::StrongMemoize + attr_reader :pipeline def initialize(pipeline) @@ -15,6 +17,26 @@ module Gitlab raise NotImplementedError end + def project + pipeline.project + end + + def sha + pipeline.sha + end + + def top_level_worktree_paths + strong_memoize(:top_level_worktree_paths) do + project.repository.tree(sha).blobs.map(&:path) + end + end + + def all_worktree_paths + strong_memoize(:all_worktree_paths) do + project.repository.ls_files(sha) + end + end + protected def pipeline_attributes diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index 1d7bfba75cd..8ddcf1d523e 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -4,7 +4,7 @@ module Gitlab module Ci module Build class Image - attr_reader :alias, :command, :entrypoint, :name, :ports + attr_reader :alias, :command, :entrypoint, :name, :ports, :variables class << self def from_image(job) @@ -33,6 +33,7 @@ module Gitlab @entrypoint = image[:entrypoint] @name = image[:name] @ports = build_ports(image).select(&:valid?) + @variables = build_variables(image) end end @@ -45,6 +46,12 @@ module Gitlab def build_ports(image) image[:ports].to_a.map { |port| ::Gitlab::Ci::Build::Port.new(port) } end + + def build_variables(image) + image[:variables].to_a.map do |key, value| + { key: key, value: value.to_s } + end + end end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index 85e77438f51..e2b54797dc8 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -15,19 +15,21 @@ module Gitlab @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) end - def satisfied_by?(pipeline, context) - paths = worktree_paths(pipeline) + def satisfied_by?(_pipeline, context) + paths = worktree_paths(context) exact_matches?(paths) || pattern_matches?(paths) end private - def worktree_paths(pipeline) + def worktree_paths(context) + return unless context.project + if @top_level_only - pipeline.top_level_worktree_paths + context.top_level_worktree_paths else - pipeline.all_worktree_paths + context.all_worktree_paths end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index aceaf012f7e..6f149385969 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -19,11 +19,12 @@ module Gitlab attr_reader :root, :context, :source_ref_path, :source - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil) - @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path) + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil) + @source_ref_path = pipeline&.source_ref_path + + @context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) @context.set_deadline(TIMEOUT_SECONDS) - @source_ref_path = source_ref_path @source = source @config = expand_config(config) @@ -108,16 +109,16 @@ module Gitlab end end - def build_context(project:, sha:, user:, parent_pipeline:, ref:) + def build_context(project:, pipeline:, sha:, user:, parent_pipeline:) Config::External::Context.new( project: project, sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: build_variables(project: project, ref: ref)) + variables: build_variables(project: project, pipeline: pipeline)) end - def build_variables(project:, ref:) + def build_variables(project:, pipeline:) Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project @@ -126,18 +127,12 @@ module Gitlab # # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence variables.concat(project.predefined_variables) - variables.concat(pipeline_predefined_variables(ref: ref)) - variables.concat(project.ci_instance_variables_for(ref: ref)) - variables.concat(project.group.ci_variables_for(ref, project)) if project.group - variables.concat(project.ci_variables_for(ref: ref)) - end - end - - # https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables - # to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines. - def pipeline_predefined_variables(ref:) - Gitlab::Ci::Variables::Collection.new.tap do |v| - v.append(key: 'CI_COMMIT_REF_NAME', value: ref) + variables.concat(pipeline.predefined_variables) if pipeline + variables.concat(project.ci_instance_variables_for(ref: source_ref_path)) + variables.concat(project.group.ci_variables_for(source_ref_path, project)) if project.group + variables.concat(project.ci_variables_for(ref: source_ref_path)) + variables.concat(pipeline.variables) if pipeline + variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline&.pipeline_schedule end end diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb index d3d0f098814..fa99a7204d6 100644 --- a/lib/gitlab/ci/config/entry/include/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -9,9 +9,9 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[if].freeze + ALLOWED_KEYS = %i[if exists].freeze - attributes :if + attributes :if, :exists validations do validates :config, presence: true diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index f867189d521..75bbe2ccb1b 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,10 +14,10 @@ module Gitlab ALLOWED_KEYS = %i[tags script type image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout - release dast_configuration secrets].freeze + release].freeze validations do - validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS + validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS validates :script, presence: true with_options allow_nil: true do @@ -178,6 +178,10 @@ module Gitlab allow_failure_defined? ? static_allow_failure : manual_action? end + def self.allowed_keys + ALLOWED_KEYS + end + private def allow_failure_criteria diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 2549c35ebd6..520b1ce6119 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -23,6 +23,7 @@ module Gitlab validates :config, presence: true validates :name, presence: true validates :name, type: Symbol + validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) } validates :config, disallowed_keys: { in: %i[only except when start_in], diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 247bf930d3b..f27dca4986e 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -15,7 +15,7 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name entrypoint command alias ports].freeze + ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze validations do validates :config, hash_or_string: true @@ -32,6 +32,10 @@ module Gitlab entry :ports, Entry::Ports, description: 'Ports used to expose the service' + entry :variables, ::Gitlab::Ci::Config::Entry::Variables, + description: 'Environment variables available for this service.', + inherit: false + attributes :ports def alias diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index e0adb1b19c2..51624dc30ea 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -5,6 +5,8 @@ module Gitlab class Config module External class Context + include Gitlab::Utils::StrongMemoize + TimeoutError = Class.new(StandardError) attr_reader :project, :sha, :user, :parent_pipeline, :variables @@ -22,6 +24,18 @@ module Gitlab yield self if block_given? end + def top_level_worktree_paths + strong_memoize(:top_level_worktree_paths) do + project.repository.tree(sha).blobs.map(&:path) + end + end + + def all_worktree_paths + strong_memoize(:all_worktree_paths) do + project.repository.ls_files(sha) + end + end + def mutate(attrs = {}) self.class.new(**attrs) do |ctx| ctx.expandset = expandset diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 1cf4f252ab9..0c969daf7fd 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -33,8 +33,7 @@ module Gitlab report_data rescue JSON::ParserError raise SecurityReportParserError, 'JSON parsing failed' - rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + rescue StandardError raise SecurityReportParserError, "#{report.type} security report parsing failed" end @@ -115,7 +114,7 @@ module Gitlab flags: flags, links: links, remediations: remediations, - raw_metadata: data.to_json, + original_data: data, metadata_version: report_version, details: data['details'] || {}, signatures: signatures, diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 143b930c669..73cfa02ce4b 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -34,7 +34,7 @@ module Gitlab end def file_name - "#{report_type}.json" + "#{report_type.to_s.dasherize}-report-format.json" end end diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json b/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json index a7159be0190..a7159be0190 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json b/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json index 462e23a151c..462e23a151c 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index c9bc4ec411d..beb8801096b 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,3 +1,4 @@ +# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab @@ -144,3 +145,5 @@ module Gitlab end end end + +# rubocop:enable Naming/FileName diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 5251dd3d40a..f3c937ddd28 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -14,7 +14,7 @@ module Gitlab result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, - source_ref_path: @pipeline.source_ref_path, + pipeline: @pipeline, sha: @pipeline.sha, source: @pipeline.source, user: current_user, diff --git a/lib/gitlab/ci/pipeline/chain/create_cross_database_associations.rb b/lib/gitlab/ci/pipeline/chain/create_cross_database_associations.rb new file mode 100644 index 00000000000..bb5b4e722b7 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create_cross_database_associations.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CreateCrossDatabaseAssociations < Chain::Base + def perform! + # to be overridden in EE + end + + def break? + false # to be overridden in EE + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::CreateCrossDatabaseAssociations.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::CreateCrossDatabaseAssociations') diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 321efa7854f..b5e48f210ad 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -51,6 +51,15 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_builder_scoped_variables_histogram + name = :gitlab_ci_pipeline_builder_scoped_variables_duration + comment = 'Pipeline variables builder scoped_variables duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5, 10, 30, 60, 120] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + def self.pipeline_processing_events_counter name = :gitlab_ci_pipeline_processing_events_total comment = 'Total amount of pipeline processing events' diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 9ad5d6538b7..72837b8ec22 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,11 +11,11 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(context, attributes, previous_stages, current_stage) + def initialize(context, attributes, stages_for_needs_lookup = []) @context = context @pipeline = context.pipeline @seed_attributes = attributes - @stages_for_needs_lookup = (previous_stages + [current_stage]).compact + @stages_for_needs_lookup = stages_for_needs_lookup.compact @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @@ -90,7 +90,7 @@ module Gitlab ::Ci::Bridge.new(attributes) else ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.environment_attributes_for(build)) + build.assign_attributes(self.class.deployment_attributes_for(build)) end end end @@ -101,10 +101,10 @@ module Gitlab .to_resource end - def self.environment_attributes_for(build) + def self.deployment_attributes_for(build, environment = nil) return {} unless build.has_environment? - environment = Seed::Environment.new(build).to_resource + environment = Seed::Environment.new(build).to_resource if environment.nil? unless environment.persisted? if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) && @@ -173,7 +173,7 @@ module Gitlab end def variable_expansion_errors - expanded_collection = evaluate_context.variables.sort_and_expand_all(@pipeline.project) + expanded_collection = evaluate_context.variables.sort_and_expand_all errors = expanded_collection.errors ["#{name}: #{errors}"] if errors end @@ -244,5 +244,3 @@ module Gitlab end end end - -Gitlab::Ci::Pipeline::Seed::Build.prepend_mod_with('Gitlab::Ci::Pipeline::Seed::Build') diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 018fb260986..bc56fe9bef9 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -17,7 +17,7 @@ module Gitlab @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(context, attributes, previous_stages, self) + Seed::Build.new(context, attributes, previous_stages + [self]) end end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 39531e12f69..47ec82ac86c 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -17,7 +17,6 @@ module Gitlab attr_reader :name attr_reader :old_location attr_reader :project_fingerprint - attr_reader :raw_metadata attr_reader :report_type attr_reader :scanner attr_reader :scan @@ -28,10 +27,13 @@ module Gitlab attr_reader :details attr_reader :signatures attr_reader :project_id + attr_reader :original_data delegate :file_path, :start_line, :end_line, to: :location - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + alias_method :cve, :compare_key + + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers @@ -40,7 +42,7 @@ module Gitlab @location = location @metadata_version = metadata_version @name = name - @raw_metadata = raw_metadata + @original_data = original_data @report_type = report_type @scanner = scanner @scan = scan @@ -74,6 +76,10 @@ module Gitlab uuid details signatures + description + message + cve + solution ].each_with_object({}) do |key, hash| hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend end @@ -88,8 +94,8 @@ module Gitlab @location = new_location end - def unsafe?(severity_levels) - severity.in?(severity_levels) + def unsafe?(severity_levels, report_types) + severity.to_s.in?(severity_levels) && (report_types.blank? || report_type.to_s.in?(report_types) ) end def eql?(other) @@ -141,6 +147,30 @@ module Gitlab scanner <=> other.scanner end + def has_signatures? + signatures.present? + end + + def raw_metadata + @raw_metadata ||= original_data.to_json + end + + def description + original_data['description'] + end + + def message + original_data['message'] + end + + def solution + original_data['solution'] + end + + def location_data + original_data['location'] + end + private def generate_project_fingerprint diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 1ba2d909d99..417319cb5be 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -69,6 +69,10 @@ module Gitlab primary_scanner <=> other.primary_scanner end + + def has_signatures? + findings.any?(&:has_signatures?) + end end end end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb index b7a5e36b108..b6372349f68 100644 --- a/lib/gitlab/ci/reports/security/reports.rb +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -22,21 +22,24 @@ module Gitlab reports.values.flat_map(&:findings) end - def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels) - unsafe_findings_count(target_reports, severity_levels) > vulnerabilities_allowed + def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types = []) + unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) > vulnerabilities_allowed end - private - - def findings_diff(target_reports) - findings - target_reports&.findings.to_a + def unsafe_findings_uuids(severity_levels, report_types) + findings.select { |finding| finding.unsafe?(severity_levels, report_types) }.map(&:uuid) end - def unsafe_findings_count(target_reports, severity_levels) - findings_diff(target_reports).count {|finding| finding.unsafe?(severity_levels)} + private + + def unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) + new_uuids = unsafe_findings_uuids(severity_levels, report_types) - target_reports&.unsafe_findings_uuids(severity_levels, report_types).to_a + new_uuids.count end end end end end end + +Gitlab::Ci::Reports::Security::Reports.prepend_mod_with('Gitlab::Ci::Reports::Security::Reports') diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index adb5d430d46..89fd59d98f4 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -179,3 +179,11 @@ include: - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml + +# The latest build job generates a dotenv report artifact with a CI_APPLICATION_TAG +# that also includes the image digest. This configures Auto Deploy to receive +# this artifact and use the updated CI_APPLICATION_TAG for deployments. +.auto-deploy: + dependencies: [build] +dast_environment_deploy: + dependencies: [build] diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index f147ad9332d..426076c84a1 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -1,54 +1,76 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Django.gitlab-ci.yml - -# Official framework image. Look for the different tagged releases at: -# https://hub.docker.com/r/library/python -image: python:latest - -# Pick zero or more services to be used on all builds. -# Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service -services: - - mysql:latest - - postgres:latest +# This example is for testing Django with MySQL. +# +# The test CI/CD variables MYSQL_DB, MYSQL_USER and MYSQL_PASS can be set in the project settings at: +# Settings --> CI/CD --> Variables +# +# The Django settings in settings.py, used in tests, might look similar to: +# +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.mysql', +# 'NAME': os.environ.get('MYSQL_DATABASE'), +# 'USER': os.environ.get('MYSQL_USER'), +# 'PASSWORD': os.environ.get('MYSQL_PASSWORD'), +# 'HOST': 'mysql', +# 'PORT': '3306', +# 'CONN_MAX_AGE':60, +# }, +# } +# +# It is possible to use '--settings' to specify a custom settings file on the command line below or use an environment +# variable to trigger an include on the bottom of your settings.py: +# if os.environ.get('DJANGO_CONFIG')=='test': +# from .settings_test import * +# +# It is also possible to hardcode the database name and credentials in the settings.py file and in the .gitlab-ci.yml file. +# +# The mysql service needs some variables too. See https://hub.docker.com/_/mysql for possible mysql env variables +# Note that when using a service in GitLab CI/CD that needs environment variables to run, only variables defined in +# .gitlab-ci.yml are passed to the service and variables defined in the GitLab UI are not. +# https://gitlab.com/gitlab-org/gitlab/-/issues/30178 variables: - POSTGRES_DB: database_name + # DJANGO_CONFIG: "test" + MYSQL_DATABASE: $MYSQL_DB + MYSQL_ROOT_PASSWORD: $MYSQL_PASS + MYSQL_USER: $MYSQL_USER + MYSQL_PASSWORD: $MYSQL_PASS -# This folder is cached between builds -# https://docs.gitlab.com/ee/ci/yaml/index.html#cache -cache: - paths: - - ~/.cache/pip/ +default: + image: ubuntu:20.04 + # + # Pick zero or more services to be used on all builds. + # Only needed when using a docker container to run your tests in. + # Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service + services: + - mysql:8.0 + # + # This folder is cached between builds + # http://docs.gitlab.com/ee/ci/yaml/README.html#cache + cache: + paths: + - ~/.cache/pip/ + before_script: + - apt -y update + - apt -y install apt-utils + - apt -y install net-tools python3.8 python3-pip mysql-client libmysqlclient-dev + - apt -y upgrade + - pip3 install -r requirements.txt -# This is a basic example for a gem or script which doesn't use -# services such as redis or postgres -before_script: - - python -V # Print out python version for debugging - # Uncomment next line if your Django app needs a JS runtime: - # - apt-get update -q && apt-get install nodejs -yqq - - pip install -r requirements.txt -# To get Django tests to work you may need to create a settings file using -# the following DATABASES: -# -# DATABASES = { -# 'default': { -# 'ENGINE': 'django.db.backends.postgresql_psycopg2', -# 'NAME': 'ci', -# 'USER': 'postgres', -# 'PASSWORD': 'postgres', -# 'HOST': 'postgres', -# 'PORT': '5432', -# }, -# } -# -# and then adding `--settings app.settings.ci` (or similar) to the test command +migrations: + stage: build + script: + - python3 manage.py makemigrations + # - python3 manage.py makemigrations myapp + - python3 manage.py migrate + - python3 manage.py check + -test: - variables: - DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" +django-tests: + stage: test script: - - python manage.py test + # The MYSQL user only gets permissions for MYSQL_DB, so Django can't create a test database. + - echo "GRANT ALL on *.* to '${MYSQL_USER}';"| mysql -u root --password="${MYSQL_ROOT_PASSWORD}" -h mysql + # use python3 explicitly. see https://wiki.ubuntu.com/Python/3 + - python3 manage.py test diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 56899614cc6..99fd9870b1d 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -70,7 +70,7 @@ browser_performance: reports: browser_performance: browser-performance.json rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$BROWSER_PERFORMANCE_DISABLED' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 56899614cc6..99fd9870b1d 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -70,7 +70,7 @@ browser_performance: reports: browser_performance: browser-performance.json rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$BROWSER_PERFORMANCE_DISABLED' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 6a3b0cfa9e7..211adc9bd5b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -3,7 +3,7 @@ # This template is scheduled for removal when testing is complete: https://gitlab.com/gitlab-org/gitlab/-/issues/337987 variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.3.1' + AUTO_BUILD_IMAGE_VERSION: 'v1.5.0' build: stage: build @@ -23,6 +23,9 @@ build: export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} fi - /build/build.sh + artifacts: + reports: + dotenv: gl-auto-build-variables.env rules: - if: '$BUILD_DISABLED' when: never diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml index 31ca68c57d7..11f8376f0b4 100644 --- a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml @@ -9,6 +9,6 @@ cloud_formation: rules: - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") || ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 65a58130962..28ac627f103 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.17.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" @@ -10,6 +10,7 @@ dast_environment_deploy: script: - auto-deploy check_kube_domain - auto-deploy download_chart + - auto-deploy use_kube_context || true - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret @@ -29,7 +30,7 @@ dast_environment_deploy: - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given when: never - if: $CI_COMMIT_BRANCH && - $CI_KUBERNETES_ACTIVE && + ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ stop_dast_environment: @@ -38,6 +39,7 @@ stop_dast_environment: variables: GIT_STRATEGY: none script: + - auto-deploy use_kube_context || true - auto-deploy initialize_tiller - auto-deploy delete environment: @@ -52,6 +54,6 @@ stop_dast_environment: - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given when: never - if: $CI_COMMIT_BRANCH && - $CI_KUBERNETES_ACTIVE && + ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ when: always diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 58f13746a1f..973db26bf2d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.17.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" @@ -11,6 +11,7 @@ review: script: - auto-deploy check_kube_domain - auto-deploy download_chart + - auto-deploy use_kube_context || true - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret @@ -24,7 +25,7 @@ review: paths: [environment_url.txt, tiller.log] when: always rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never @@ -38,6 +39,7 @@ stop_review: variables: GIT_STRATEGY: none script: + - auto-deploy use_kube_context || true - auto-deploy initialize_tiller - auto-deploy delete environment: @@ -45,7 +47,7 @@ stop_review: action: stop allow_failure: true rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never @@ -66,6 +68,7 @@ staging: script: - auto-deploy check_kube_domain - auto-deploy download_chart + - auto-deploy use_kube_context || true - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret @@ -74,7 +77,7 @@ staging: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never @@ -91,6 +94,7 @@ canary: script: - auto-deploy check_kube_domain - auto-deploy download_chart + - auto-deploy use_kube_context || true - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret @@ -101,7 +105,7 @@ canary: rules: - if: '$CI_DEPLOY_FREEZE != null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never @@ -114,6 +118,7 @@ canary: script: - auto-deploy check_kube_domain - auto-deploy download_chart + - auto-deploy use_kube_context || true - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret @@ -132,7 +137,7 @@ production: rules: - if: '$CI_DEPLOY_FREEZE != null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$STAGING_ENABLED' when: never @@ -150,7 +155,7 @@ production_manual: rules: - if: '$CI_DEPLOY_FREEZE != null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$INCREMENTAL_ROLLOUT_ENABLED' when: never @@ -168,6 +173,7 @@ production_manual: script: - auto-deploy check_kube_domain - auto-deploy download_chart + - auto-deploy use_kube_context || true - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret @@ -188,7 +194,7 @@ production_manual: rules: - if: '$CI_DEPLOY_FREEZE != null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never @@ -203,7 +209,7 @@ production_manual: rules: - if: '$CI_DEPLOY_FREEZE != null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 530ab1d0f99..248040b8b18 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -21,7 +21,7 @@ review: paths: [environment_url.txt, tiller.log] when: always rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never @@ -42,7 +42,7 @@ stop_review: action: stop allow_failure: true rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never @@ -71,7 +71,7 @@ staging: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never @@ -96,7 +96,7 @@ canary: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never @@ -125,7 +125,7 @@ canary: production: <<: *production_template rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$STAGING_ENABLED' when: never @@ -141,7 +141,7 @@ production_manual: <<: *production_template allow_failure: false rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$INCREMENTAL_ROLLOUT_ENABLED' when: never @@ -177,7 +177,7 @@ production_manual: resource_group: production allow_failure: true rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never @@ -190,7 +190,7 @@ production_manual: .timed_rollout_template: &timed_rollout_template <<: *rollout_template rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml index 7efbcab221b..ab3bc511cba 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -16,7 +16,7 @@ review_ec2: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$REVIEW_DISABLED' when: never @@ -32,7 +32,7 @@ production_ec2: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 332c58c8695..9bb2ba69d84 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -42,7 +42,7 @@ review_ecs: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$REVIEW_DISABLED' when: never @@ -58,7 +58,7 @@ stop_review_ecs: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$REVIEW_DISABLED' when: never @@ -77,7 +77,7 @@ review_fargate: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$REVIEW_DISABLED' when: never @@ -93,7 +93,7 @@ stop_review_fargate: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$REVIEW_DISABLED' when: never @@ -107,7 +107,7 @@ production_ecs: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never @@ -118,7 +118,7 @@ production_fargate: rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"' when: never - - if: '$CI_KUBERNETES_ACTIVE' + - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml index 1ec1aa60d88..d55c126eeb7 100644 --- a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml @@ -72,7 +72,7 @@ rules: - if: '$MIGRATE_HELM_2TO3 != "true"' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never @@ -89,7 +89,7 @@ review:helm-2to3:cleanup: rules: - if: '$MIGRATE_HELM_2TO3 != "true" && $CLEANUP_HELM_2TO3 == null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never @@ -104,7 +104,7 @@ review:helm-2to3:cleanup: rules: - if: '$MIGRATE_HELM_2TO3 != "true"' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never @@ -119,7 +119,7 @@ staging:helm-2to3:cleanup: rules: - if: '$MIGRATE_HELM_2TO3 != "true" && $CLEANUP_HELM_2TO3 == null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never @@ -132,7 +132,7 @@ staging:helm-2to3:cleanup: rules: - if: '$MIGRATE_HELM_2TO3 != "true"' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: manual @@ -145,7 +145,7 @@ production:helm-2to3:cleanup: rules: - if: '$MIGRATE_HELM_2TO3 != "true" && $CLEANUP_HELM_2TO3 == null' when: never - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: manual diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 9a7c513c25f..8e34388893a 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -23,7 +23,7 @@ load_performance: reports: load_performance: load-performance.json rules: - - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")' when: never - if: '$LOAD_PERFORMANCE_DISABLED' when: never diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml new file mode 100644 index 00000000000..b763705857e --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -0,0 +1,34 @@ +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + +iac-sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + variables: + SEARCH_MAX_DEPTH: 4 + allow_failure: true + script: + - /analyzer run + +kics-iac-sast: + extends: iac-sast + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 0 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ + when: never + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml new file mode 100644 index 00000000000..f1b1c20b4e0 --- /dev/null +++ b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml @@ -0,0 +1,47 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml + +# Build and publish a tag/branch to Gitlab Docker Registry using Kaniko and Gitlab Docker executor. +# Kaniko can build Docker images without using Docker-In-Docker and it's permission +# drawbacks. No additional configuration required. +kaniko-build: + variables: + # Additional options for Kaniko executor. + # For more details see https://github.com/GoogleContainerTools/kaniko/blob/master/README.md#additional-flags + KANIKO_ARGS: "" + stage: build + image: + # For latest releases see https://github.com/GoogleContainerTools/kaniko/releases + # Only debug/*-debug versions of the Kaniko image are known to work within Gitlab CI + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + # Compose docker tag name + # Git Branch/Tag to Docker Image Tag Mapping + # * Default Branch: main -> latest + # * Branch: feature/my-feature -> branch-feature-my-feature + # * Tag: v1.0.0/beta2 -> v1.0.0-beta2 + - | + if [ "$CI_COMMIT_REF_NAME" = $CI_DEFAULT_BRANCH ]; then + VERSION="latest" + elif [ -n "$CI_COMMIT_TAG" ];then + NOSLASH=$(echo "$CI_COMMIT_TAG" | tr -s / - ) + SANITIZED="${NOSLASH//[^a-zA-Z0-9\-\.]/}" + VERSION="$SANITIZED" + else \ + NOSLASH=$(echo "$CI_COMMIT_REF_NAME" | tr -s / - ) + SANITIZED="${NOSLASH//[^a-zA-Z0-9\-]/}" + VERSION="branch-$SANITIZED" + fi + - echo $VERSION + - mkdir -p /kaniko/.docker + # Write credentials to access Gitlab Container Registry within the runner/ci + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + # Build and push the container. To disable push add --no-push + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$VERSION $KANIKO_ARGS + # Run this job in a branch/tag where a Dockerfile exists + rules: + - exists: + - Dockerfile diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index ceeefa8aea6..544774d3b06 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -1,7 +1,7 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.lastest.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ # diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml index ed4876c2bcc..6b861510eef 100644 --- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml @@ -12,7 +12,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables variables: - CIS_ANALYZER_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/cluster-image-scanning:0 + CIS_ANALYZER_IMAGE: registry.gitlab.com/security-products/cluster-image-scanning:0 cluster_image_scanning: image: "$CIS_ANALYZER_IMAGE" diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 0802868d67f..0ecbe5e14b8 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -51,7 +51,7 @@ dast: $REVIEW_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $CI_KUBERNETES_ACTIVE && + ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index ac7d87a4cda..3d07674c377 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -1,7 +1,7 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.lastest.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml # To use this template, add the following to your .gitlab-ci.yml file: # @@ -52,7 +52,7 @@ dast: $DAST_API_SPECIFICATION == null when: never - if: $CI_COMMIT_BRANCH && - $CI_KUBERNETES_ACTIVE && + ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index aa7b394a13c..197ce2438e6 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -74,6 +74,9 @@ gemnasium-maven-dependency_scanning: # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" + # Stop reporting Gradle as "maven". + # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 + DS_REPORT_PACKAGE_MANAGER_MAVEN_WHEN_JAVA: "false" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never @@ -97,6 +100,9 @@ gemnasium-python-dependency_scanning: # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" + # Stop reporting Pipenv and Setuptools as "pip". + # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 + DS_REPORT_PACKAGE_MANAGER_PIP_WHEN_PYTHON: "false" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml new file mode 100644 index 00000000000..8c0d72ff282 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml @@ -0,0 +1,2 @@ +include: + template: Jobs/SAST-IaC.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 081a3a6cc78..e554742735c 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -7,20 +7,17 @@ include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: - - init - validate - build - deploy - - cleanup - -init: - extends: .terraform:init fmt: extends: .terraform:fmt + needs: [] validate: extends: .terraform:validate + needs: [] build: extends: .terraform:build diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 3a70e6bc4b8..a0ec07e61e1 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -21,18 +21,11 @@ cache: paths: - ${TF_ROOT}/.terraform/ -.terraform:init: &terraform_init - stage: init - script: - - cd ${TF_ROOT} - - gitlab-terraform init - .terraform:fmt: &terraform_fmt stage: validate - needs: [] script: - cd ${TF_ROOT} - - gitlab-terraform fmt -check -recursive + - gitlab-terraform fmt allow_failure: true .terraform:validate: &terraform_validate @@ -60,10 +53,9 @@ cache: - cd ${TF_ROOT} - gitlab-terraform apply resource_group: ${TF_STATE_NAME} - when: manual - only: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual .terraform:destroy: &terraform_destroy stage: cleanup diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 22c40d8a8b8..4f63ff93d4d 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.3 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.0.1 script: /gitlab-accessibility.sh $a11y_urls allow_failure: true artifacts: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 25075cc8f90..7d08f0230fc 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -78,7 +78,7 @@ module Gitlab end def archived_trace_exist? - trace_artifact&.exists? + archived? end def live_trace_exist? @@ -156,7 +156,7 @@ module Gitlab def read_stream stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact + if archived? trace_artifact.open elsif job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) @@ -174,7 +174,7 @@ module Gitlab def unsafe_write!(mode, &blk) stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact + if archived? raise AlreadyArchivedError, 'Could not write to the archived trace' elsif current_path File.open(current_path, mode) @@ -195,7 +195,7 @@ module Gitlab def unsafe_archive! raise ArchiveError, 'Job is not finished yet' unless job.complete? - already_archived?.tap do |archived| + archived?.tap do |archived| destroy_any_orphan_trace_data! raise AlreadyArchivedError, 'Could not archive again' if archived end @@ -218,7 +218,7 @@ module Gitlab end end - def already_archived? + def archived? # TODO check checksum to ensure archive completed successfully # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619 trace_artifact&.archived_trace_exists? @@ -227,11 +227,12 @@ module Gitlab def destroy_any_orphan_trace_data! return unless trace_artifact - if already_archived? - # An archive already exists, so make sure to remove the trace chunks + if archived? + # An archive file exists, so remove the trace chunks erase_trace_chunks! else - # An archive already exists, but its associated file does not, so remove it + # A trace artifact record exists with no archive file + # but an archive was attempted, so cleanup the associated record trace_artifact.destroy! end end diff --git a/lib/gitlab/ci/trace/archive.rb b/lib/gitlab/ci/trace/archive.rb index 5047cf04562..d4a451ca526 100644 --- a/lib/gitlab/ci/trace/archive.rb +++ b/lib/gitlab/ci/trace/archive.rb @@ -62,7 +62,7 @@ module Gitlab trace_metadata.update!(remote_checksum: remote_checksum) unless trace_metadata.remote_checksum_valid? - metrics.increment_error_counter(type: :archive_invalid_checksum) + metrics.increment_error_counter(error_reason: :archive_invalid_checksum) end end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 174a5f184ff..f3ded3cda4a 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -21,7 +21,7 @@ module Gitlab :corrupted # malformed trace found after comparing CRC32 and size ].freeze - TRACE_ERROR_TYPES = [ + TRACE_ERROR_REASONS = [ :chunks_invalid_size, # used to be :corrupted :chunks_invalid_checksum, # used to be :invalid :archive_invalid_checksum # malformed trace found into object store after comparing MD5 @@ -39,12 +39,12 @@ module Gitlab self.class.trace_bytes.increment({}, size.to_i) end - def increment_error_counter(type: :unknown) - unless TRACE_ERROR_TYPES.include?(type) - raise ArgumentError, "unknown error type: #{type}" + def increment_error_counter(error_reason: :unknown) + unless TRACE_ERROR_REASONS.include?(error_reason) + raise ArgumentError, "unknown error reason: #{error_reason}" end - self.class.trace_errors_counter.increment(type: type) + self.class.trace_errors_counter.increment(error_reason: error_reason) end def observe_migration_duration(seconds) diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb new file mode 100644 index 00000000000..f4c5a06af97 --- /dev/null +++ b/lib/gitlab/ci/variables/builder.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + include ::Gitlab::Utils::StrongMemoize + + def initialize(pipeline) + @pipeline = pipeline + end + + def scoped_variables(job, environment:, dependencies:) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.concat(predefined_variables(job)) if pipeline.predefined_vars_in_builder_enabled? + end + end + + private + + attr_reader :pipeline + + def predefined_variables(job) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_JOB_NAME', value: job.name) + variables.append(key: 'CI_JOB_STAGE', value: job.stage) + variables.append(key: 'CI_JOB_MANUAL', value: 'true') if job.action? + variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if job.trigger_request + + variables.append(key: 'CI_NODE_INDEX', value: job.options[:instance].to_s) if job.options&.include?(:instance) + variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value(job).to_s) + + # legacy variables + variables.append(key: 'CI_BUILD_NAME', value: job.name) + variables.append(key: 'CI_BUILD_STAGE', value: job.stage) + variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request + variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? + end + end + + def ci_node_total_value(job) + parallel = job.options&.dig(:parallel) + parallel = parallel.dig(:total) if parallel.is_a?(Hash) + parallel || 1 + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 09c75a2b3f1..a00c1da97ea 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -89,9 +89,7 @@ module Gitlab end end - def sort_and_expand_all(project, keep_undefined: false) - return self if Feature.disabled?(:variable_inside_variable, project, default_enabled: :yaml) - + def sort_and_expand_all(keep_undefined: false) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index a97c7050fbb..6215ba40ebe 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -80,7 +80,6 @@ module Gitlab cache: job[:cache], resource_group_key: job[:resource_group], scheduling_type: job[:scheduling_type], - secrets: job[:secrets], options: { image: job[:image], services: job[:services], |