diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 20:02:30 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 20:02:30 +0000 |
| commit | 41fe97390ceddf945f3d967b8fdb3de4c66b7dea (patch) | |
| tree | 9c8d89a8624828992f06d892cd2f43818ff5dcc8 /lib/gitlab/ci | |
| parent | 0804d2dc31052fb45a1efecedc8e06ce9bc32862 (diff) | |
| download | gitlab-ce-41fe97390ceddf945f3d967b8fdb3de4c66b7dea.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
62 files changed, 578 insertions, 277 deletions
diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 7ade9ca5085..2e5f6611e73 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -36,7 +36,7 @@ module Gitlab # the pattern matching does not work for merge requests pipelines if pipeline.branch? || pipeline.tag? regexp = Gitlab::UntrustedRegexp::RubySyntax - .fabricate(pattern, fallback: true, project: pipeline.project) + .fabricate(pattern, project: pipeline.project) if regexp regexp.match?(pipeline.ref) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 8dd1f686132..06c81fd65dd 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -37,10 +37,12 @@ module Gitlab next unless dependencies.present? next unless needs_value.present? - missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) + if needs_value[:job].nil? && needs_value[:cross_dependency].present? + errors.add(:needs, "corresponding to dependencies must be from the same pipeline") + else + missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) - if missing_needs.any? - errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") + errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any? end end end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 7b14218d3ea..adc3660d950 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -17,7 +17,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable validations do - validates :config, array_of_strings_or_regexps_with_fallback: true + validates :config, array_of_strings_or_regexps: true end def value @@ -38,7 +38,7 @@ module Gitlab validate :variables_expressions_syntax with_options allow_nil: true do - validates :refs, array_of_strings_or_regexps_with_fallback: true + validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] validates :variables, array_of_strings: true validates :changes, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index e45dbfa243f..f8fce1abc06 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -8,6 +8,7 @@ module Gitlab # Entry that represents a configuration of job artifacts. # class Reports < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable @@ -15,10 +16,13 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications - requirements coverage_fuzzing api_fuzzing cluster_image_scanning].freeze + requirements coverage_fuzzing api_fuzzing cluster_image_scanning + coverage_report].freeze attributes ALLOWED_KEYS + entry :coverage_report, Reports::CoverageReport, description: 'Coverage report configuration.' + validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS @@ -47,10 +51,18 @@ module Gitlab validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end + + validates :config, mutually_exclusive_keys: [:coverage_report, :cobertura] end def value - @config.transform_values { |v| Array(v) } + @config.transform_values do |value| + if value.is_a?(Hash) + value + else + Array(value) + end + end end end end diff --git a/lib/gitlab/ci/config/entry/reports/coverage_report.rb b/lib/gitlab/ci/config/entry/reports/coverage_report.rb new file mode 100644 index 00000000000..98119c7fd53 --- /dev/null +++ b/lib/gitlab/ci/config/entry/reports/coverage_report.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Reports + class CoverageReport < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[coverage_format path].freeze + SUPPORTED_COVERAGE = %w[cobertura].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + with_options(presence: true) do + validates :coverage_format, inclusion: { in: SUPPORTED_COVERAGE, message: "must be one of supported formats: #{SUPPORTED_COVERAGE.join(', ')}." } + validates :path, type: String + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 840f2d6f31a..4722f2e9a61 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -24,7 +24,7 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay? validates :start_in, presence: true, if: :specifies_delay? - validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay? + validates :start_in, duration: { limit: '1 week' }, if: :specifies_delay? with_options allow_nil: true do validates :if, expression: true diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index c6ba53adfd7..0f94b3f94fe 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -5,12 +5,13 @@ module Gitlab class Config module Entry ## - # Entry that represents a cross-project downstream trigger. + # Entry that represents a parent-child or cross-project downstream trigger. # class Trigger < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleTrigger, if: -> (config) { config.is_a?(String) } strategy :ComplexTrigger, if: -> (config) { config.is_a?(Hash) } + # cross-project class SimpleTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -28,11 +29,13 @@ module Gitlab config.key?(:include) end + # cross-project class CrossProjectTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[project branch strategy].freeze + ALLOWED_KEYS = %i[project branch strategy forward].freeze attributes :project, :branch, :strategy validations do @@ -42,15 +45,26 @@ module Gitlab validates :branch, type: String, allow_nil: true validates :strategy, type: String, inclusion: { in: %w[depend], message: 'should be depend' }, allow_nil: true end + + entry :forward, ::Gitlab::Ci::Config::Entry::Trigger::Forward, + description: 'List what to forward to downstream pipelines' + + def value + { project: project, + branch: branch, + strategy: strategy, + forward: forward_value }.compact + end end + # parent-child class SameProjectTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable INCLUDE_MAX_SIZE = 3 - ALLOWED_KEYS = %i[strategy include].freeze + ALLOWED_KEYS = %i[strategy include forward].freeze attributes :strategy validations do @@ -64,8 +78,13 @@ module Gitlab reserved: true, metadata: { max_size: INCLUDE_MAX_SIZE } + entry :forward, ::Gitlab::Ci::Config::Entry::Trigger::Forward, + description: 'List what to forward to downstream pipelines' + def value - @config + { include: @config[:include], + strategy: strategy, + forward: forward_value }.compact end end diff --git a/lib/gitlab/ci/config/entry/trigger/forward.rb b/lib/gitlab/ci/config/entry/trigger/forward.rb new file mode 100644 index 00000000000..f80f018f149 --- /dev/null +++ b/lib/gitlab/ci/config/entry/trigger/forward.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the configuration for passing attributes to the downstream pipeline + # + class Trigger + class Forward < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[yaml_variables pipeline_variables].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :yaml_variables, boolean: true + validates :pipeline_variables, boolean: true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index fdb3e1b00f9..3839c43bd53 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -33,6 +33,10 @@ module Gitlab def fetch_local_content context.project.repository.blob_data_at(context.sha, location) + rescue GRPC::InvalidArgument + errors.push("Sha #{context.sha} is not valid!") + + nil end override :expand_context_attrs diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb index 22822614b67..45787077c91 100644 --- a/lib/gitlab/ci/config/yaml/tags/reference.rb +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -27,7 +27,7 @@ module Gitlab override :_resolve def _resolve(resolver) - object = resolver.config.dig(*location) + object = config_at_location(resolver) value = resolver.deep_resolve(object) raise MissingReferenceError, missing_ref_error_message unless value @@ -35,6 +35,12 @@ module Gitlab value end + def config_at_location(resolver) + resolver.config.dig(*location) + rescue TypeError + raise MissingReferenceError, missing_ref_error_message + end + def missing_ref_error_message "#{data[:tag]} #{data[:seq].inspect} could not be found" end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index d6b3af674a6..6041907ef78 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -8,140 +8,8 @@ module Gitlab InvalidXMLError = Class.new(Gitlab::Ci::Parsers::ParserError) InvalidLineInformationError = Class.new(Gitlab::Ci::Parsers::ParserError) - GO_SOURCE_PATTERN = '/usr/local/go/src' - MAX_SOURCES = 100 - def parse!(xml_data, coverage_report, project_path: nil, worktree_paths: nil) - root = Hash.from_xml(xml_data) - - context = { - project_path: project_path, - paths: worktree_paths&.to_set, - sources: [] - } - - parse_all(root, coverage_report, context) - rescue Nokogiri::XML::SyntaxError - raise InvalidXMLError, "XML parsing failed" - end - - private - - def parse_all(root, coverage_report, context) - return unless root.present? - - root.each do |key, value| - parse_node(key, value, coverage_report, context) - end - end - - def parse_node(key, value, coverage_report, context) - if key == 'sources' && value && value['source'].present? - parse_sources(value['source'], context) - elsif key == 'package' - Array.wrap(value).each do |item| - parse_package(item, coverage_report, context) - end - elsif key == 'class' - # This means the cobertura XML does not have classes within package nodes. - # This is possible in some cases like in simple JS project structures - # running Jest. - Array.wrap(value).each do |item| - parse_class(item, coverage_report, context) - end - elsif value.is_a?(Hash) - parse_all(value, coverage_report, context) - elsif value.is_a?(Array) - value.each do |item| - parse_all(item, coverage_report, context) - end - end - end - - def parse_sources(sources, context) - return unless context[:project_path] && context[:paths] - - sources = Array.wrap(sources) - - # TODO: Go cobertura has a different format with how their packages - # are included in the filename. So we can't rely on the sources. - # We'll deal with this later. - return if sources.include?(GO_SOURCE_PATTERN) - - sources.each do |source| - source = build_source_path(source, context) - context[:sources] << source if source.present? - end - end - - def build_source_path(source, context) - # | raw source | extracted | - # |-----------------------------|------------| - # | /builds/foo/test/SampleLib/ | SampleLib/ | - # | /builds/foo/test/something | something | - # | /builds/foo/test/ | nil | - # | /builds/foo/test | nil | - source.split("#{context[:project_path]}/", 2)[1] - end - - def parse_package(package, coverage_report, context) - classes = package.dig('classes', 'class') - return unless classes.present? - - matched_filenames = Array.wrap(classes).map do |item| - parse_class(item, coverage_report, context) - end - - # Remove these filenames from the paths to avoid conflict - # with other packages that may contain the same class filenames - remove_matched_filenames(matched_filenames, context) - end - - def remove_matched_filenames(filenames, context) - return unless context[:paths] - - filenames.each { |f| context[:paths].delete(f) } - end - - def parse_class(file, coverage_report, context) - return unless file["filename"].present? && file["lines"].present? - - parsed_lines = parse_lines(file["lines"]) - filename = determine_filename(file["filename"], context) - - coverage_report.add_file(filename, Hash[parsed_lines]) if filename - - filename - end - - def parse_lines(lines) - line_array = Array.wrap(lines["line"]) - - line_array.map do |line| - # Using `Integer()` here to raise exception on invalid values - [Integer(line["number"]), Integer(line["hits"])] - end - rescue StandardError - raise InvalidLineInformationError, "Line information had invalid values" - end - - def determine_filename(filename, context) - return filename unless context[:sources].any? - - full_filename = nil - - context[:sources].each_with_index do |source, index| - break if index >= MAX_SOURCES - break if full_filename = check_source(source, filename, context) - end - - full_filename - end - - def check_source(source, filename, context) - full_path = File.join(source, filename) - - return full_path if context[:paths].include?(full_path) + Nokogiri::XML::SAX::Parser.new(SaxDocument.new(coverage_report, project_path, worktree_paths)).parse(xml_data) end end end diff --git a/lib/gitlab/ci/parsers/coverage/sax_document.rb b/lib/gitlab/ci/parsers/coverage/sax_document.rb new file mode 100644 index 00000000000..27cce0e3a3b --- /dev/null +++ b/lib/gitlab/ci/parsers/coverage/sax_document.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Coverage + class SaxDocument < Nokogiri::XML::SAX::Document + GO_SOURCE_PATTERN = '/usr/local/go/src' + MAX_SOURCES = 100 + + def initialize(coverage_report, project_path, worktree_paths) + @coverage_report = coverage_report + @project_path = project_path + @paths = worktree_paths&.to_set + + @matched_filenames = [] + @parsed_lines = [] + @sources = [] + end + + def error(error) + raise Cobertura::InvalidXMLError, "XML parsing failed with error: #{error}" + end + + def start_element(node_name, attrs = []) + return unless node_name + + self.node_name = node_name + node_attrs = Hash[attrs] + + if node_name == 'class' && node_attrs["filename"].present? + self.filename = determine_filename(node_attrs["filename"]) + self.matched_filenames << filename if filename + elsif node_name == 'line' + self.parsed_lines << parse_line(node_attrs) + end + end + + def characters(node_content) + if node_name == 'source' + parse_source(node_content) + end + end + + def end_element(node_name) + if node_name == "package" + remove_matched_filenames + elsif node_name == "class" && filename && parsed_lines.present? + coverage_report.add_file(filename, Hash[parsed_lines]) + self.filename = nil + self.parsed_lines = [] + end + end + + private + + attr_accessor :coverage_report, :project_path, :paths, :sources, :node_name, :filename, :parsed_lines, :matched_filenames + + def parse_line(line) + [Integer(line["number"]), Integer(line["hits"])] + rescue StandardError + raise Cobertura::InvalidLineInformationError, "Line information had invalid values" + end + + def parse_source(node) + return unless project_path && paths && !node.include?(GO_SOURCE_PATTERN) + + source = build_source_path(node) + self.sources << source if source.present? + end + + def build_source_path(node) + # | raw source | extracted | + # |-----------------------------|------------| + # | /builds/foo/test/SampleLib/ | SampleLib/ | + # | /builds/foo/test/something | something | + # | /builds/foo/test/ | nil | + # | /builds/foo/test | nil | + node.split("#{project_path}/", 2)[1] + end + + def remove_matched_filenames + return unless paths + + matched_filenames.each { |f| paths.delete(f) } + end + + def determine_filename(filename) + return filename unless sources.any? + + full_filename = nil + + sources.each_with_index do |source, index| + break if index >= MAX_SOURCES + break if full_filename = check_source(source, filename) + end + + full_filename + end + + def check_source(source, filename) + full_path = File.join(source, filename) + + return full_path if paths.include?(full_path) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 9aec615d012..7baae2f53d7 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -19,6 +19,8 @@ module Gitlab end def parse! + set_report_version + return report_data unless valid? raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash) @@ -26,7 +28,6 @@ module Gitlab create_scanner create_scan create_analyzer - set_report_version create_findings @@ -42,14 +43,19 @@ module Gitlab attr_reader :json_data, :report, :validate def valid? - if Feature.enabled?(:enforce_security_report_validation) - if !validate || schema_validator.valid? - report.schema_validation_status = :valid_schema - true + if Feature.enabled?(:show_report_validation_warnings, default_enabled: :yaml) + # We want validation to happen regardless of VALIDATE_SCHEMA CI variable + schema_validation_passed = schema_validator.valid? + + if validate + schema_validator.errors.each { |error| report.add_error('Schema', error) } unless schema_validation_passed + + schema_validation_passed else - report.schema_validation_status = :invalid_schema - schema_validator.errors.each { |error| report.add_error('Schema', error) } - false + # We treat all schema validation errors as warnings + schema_validator.errors.each { |error| report.add_warning('Schema', error) } + + true end else return true if !validate || schema_validator.valid? @@ -61,7 +67,7 @@ module Gitlab end def schema_validator - @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data) + @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data, report.version) end def report_data @@ -99,6 +105,7 @@ module Gitlab flags = create_flags(data['flags']) links = create_links(data['links']) location = create_location(data['location'] || {}) + evidence = create_evidence(data['evidence']) signatures = create_signatures(tracking_data(data)) if @vulnerability_finding_signatures_enabled && !signatures.empty? @@ -117,6 +124,7 @@ module Gitlab name: finding_name(data, identifiers, location), compare_key: data['cve'] || '', location: location, + evidence: evidence, severity: parse_severity_level(data['severity']), confidence: parse_confidence_level(data['confidence']), scanner: create_scanner(data['scanner']), @@ -253,6 +261,12 @@ module Gitlab raise NotImplementedError end + def create_evidence(evidence_data) + return unless evidence_data.is_a?(Hash) + + ::Gitlab::Ci::Reports::Security::Evidence.new(data: evidence_data) + end + def finding_name(data, identifiers, location) return data['message'] if data['message'].present? return data['name'] if data['name'].present? diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 651ed23eb25..0ab1a128052 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -6,20 +6,56 @@ module Gitlab module Security module Validators class SchemaValidator + # https://docs.gitlab.com/ee/update/deprecations.html#147 + SUPPORTED_VERSIONS = { + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0] + }.freeze + + # https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tags + PREVIOUS_RELEASES = %w[10.0.0 12.0.0 12.1.0 13.0.0 + 13.1.0 2.3.0-rc1 2.3.0-rc1 2.3.1-rc1 2.3.2-rc1 2.3.3-rc1 + 2.4.0-rc1 3.0.0 3.0.0-rc1 3.1.0-rc1 4.0.0-rc1 5.0.0-rc1 + 5.0.1-rc1 6.0.0-rc1 6.0.1-rc1 6.1.0-rc1 7.0.0-rc1 7.0.1-rc1 + 8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze + + # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516 + KNOWN_VERSIONS_TO_DEPRECATE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze + + VERSIONS_TO_DEPRECATE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_DEPRECATE).freeze + + DEPRECATED_VERSIONS = { + cluster_image_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + container_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + coverage_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, + dast: VERSIONS_TO_DEPRECATE_IN_15_0, + api_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, + dependency_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + sast: VERSIONS_TO_DEPRECATE_IN_15_0, + secret_detection: VERSIONS_TO_DEPRECATE_IN_15_0 + }.freeze + class Schema def root_path File.join(__dir__, 'schemas') end - def initialize(report_type) + def initialize(report_type, report_version) @report_type = report_type.to_sym + @report_version = report_version.to_s end delegate :validate, to: :schemer private - attr_reader :report_type + attr_reader :report_type, :report_version def schemer JSONSchemer.schema(pathname) @@ -30,7 +66,19 @@ module Gitlab end def schema_path - File.join(root_path, file_name) + # We can't exactly error out here pre-15.0. + # If the report itself doesn't specify the schema version, + # it will be considered invalid post-15.0 but for now we will + # validate against earliest supported version. + # https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_801479803 + # describes the indended behavior in detail + # TODO: After 15.0 - pass report_type and report_data here and + # error out if no version. + report_declared_version = File.join(root_path, report_version, file_name) + return report_declared_version if File.file?(report_declared_version) + + earliest_supported_version = SUPPORTED_VERSIONS[report_type].min + File.join(root_path, earliest_supported_version, file_name) end def file_name @@ -38,9 +86,10 @@ module Gitlab end end - def initialize(report_type, report_data) + def initialize(report_type, report_data, report_version = nil) @report_type = report_type @report_data = report_data + @report_version = report_version end def valid? @@ -53,10 +102,10 @@ module Gitlab private - attr_reader :report_type, :report_data + attr_reader :report_type, :report_data, :report_version def schema - Schema.new(report_type) + Schema.new(report_type, report_version) end end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 54b54bd0514..71dfc1a676c 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -14,7 +14,7 @@ module Gitlab with_bulk_insert_tags do pipeline.transaction do pipeline.save! - CommitStatus.bulk_insert_tags!(statuses) if bulk_insert_tags? + CommitStatus.bulk_insert_tags!(statuses) end end end @@ -29,15 +29,9 @@ module Gitlab private - def bulk_insert_tags? - strong_memoize(:bulk_insert_tags) do - ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) - end - end - def with_bulk_insert_tags previous = Thread.current['ci_bulk_insert_tags'] - Thread.current['ci_bulk_insert_tags'] = bulk_insert_tags? + Thread.current['ci_bulk_insert_tags'] = true yield ensure Thread.current['ci_bulk_insert_tags'] = previous diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 10c0fe295f8..ee6c3898592 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -94,6 +94,7 @@ module Gitlab private attr_reader :project, :destination, :started_at, :log_conditions + delegate :current_monotonic_time, to: :class def age diff --git a/lib/gitlab/ci/reports/security/evidence.rb b/lib/gitlab/ci/reports/security/evidence.rb new file mode 100644 index 00000000000..a19f52f7195 --- /dev/null +++ b/lib/gitlab/ci/reports/security/evidence.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Evidence + attr_reader :data + + def initialize(data:) + @data = data + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 69fb8474cde..911a7f5d358 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -13,6 +13,7 @@ module Gitlab attr_reader :flags attr_reader :links attr_reader :location + attr_reader :evidence attr_reader :metadata_version attr_reader :name attr_reader :old_location @@ -33,13 +34,14 @@ module Gitlab alias_method :cve, :compare_key - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers @flags = flags @links = links @location = location + @evidence = evidence @metadata_version = metadata_version @name = name @original_data = original_data @@ -65,6 +67,7 @@ module Gitlab flags links location + evidence metadata_version name project_fingerprint diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index fbf8c81ac36..8c528056d0c 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -6,7 +6,7 @@ module Gitlab module Security class Report attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers - attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status + attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings delegate :project_id, to: :pipeline @@ -19,6 +19,7 @@ module Gitlab @identifiers = {} @scanned_resources = [] @errors = [] + @warnings = [] end def commit_sha @@ -29,6 +30,10 @@ module Gitlab errors << { type: type, message: message } end + def add_warning(type, message) + warnings << { type: type, message: message } + end + def errored? errors.present? end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 287a03cefe2..7fa744d047c 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -106,7 +106,7 @@ module Gitlab private def max_tests(*used) - [DEFAULT_MAX_TESTS - used.map(&:count).sum, DEFAULT_MIN_TESTS].max + [DEFAULT_MAX_TESTS - used.sum(&:count), DEFAULT_MIN_TESTS].max end end end diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb index 59869a947a9..ac3f5838d26 100644 --- a/lib/gitlab/ci/status/build/waiting_for_approval.rb +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -9,11 +9,35 @@ module Gitlab { image: 'illustrations/manual_action.svg', size: 'svg-394', - title: 'Waiting for approval', - content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment." + title: _('Waiting for approval'), + content: _("This job deploys to the protected environment \"%{environment}\" which requires approvals.") % { environment: subject.deployment&.environment&.name } } end + def has_action? + true + end + + def action_icon + nil + end + + def action_title + nil + end + + def action_button_title + _('Go to environments page to approve or reject') + end + + def action_path + project_environments_path(subject.project) + end + + def action_method + :get + end + def self.matches?(build, user) build.waiting_for_deployment_approval? end diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 64e3b695e27..bbe1b0a4b82 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -4,8 +4,14 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml # Read more about how to use this script on this blog post https://about.gitlab.com/2019/01/28/android-publishing-with-gitlab-and-fastlane/ -# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. # If you are looking for a simpler template that does not publish, see the Android template. +# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. + +# The following environment variables also need to be defined via the CI/CD settings: +# +# - $signing_jks_file_hex: A hex-encoded Java keystore file containing your signing keys. +# To encode this file, use `xxd -p <your-keystore-file>.jks` and save the output as `$signing_jks_file_hex` +# - $google_play_service_account_api_key_json: Your Google Play service account credentials - https://docs.fastlane.tools/getting-started/android/setup/#collect-your-google-credentials stages: - environment @@ -41,20 +47,21 @@ ensureContainer: before_script: - "mkdir -p ~/.docker && echo '{\"experimental\": \"enabled\"}' > ~/.docker/config.json" - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # Skip update container `script` if the container already exists - # via https://gitlab.com/gitlab-org/gitlab-foss/issues/26866#note_97609397 -> https://stackoverflow.com/a/52077071/796832 - - docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null && exit || true - + - | + if docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null; then + echo 'Skipping job since there is already an image with this tag' + exit 0 + fi .build_job: image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG stage: build before_script: - # We store this binary file in a variable as hex with this command: `xxd -p android-app.jks` + # We store this binary file in a project variable as hex with this command: `xxd -p android-app.jks` # Then we convert the hex back to a binary file - echo "$signing_jks_file_hex" | xxd -r -p - > android-signing-keystore.jks - - "export VERSION_CODE=$CI_PIPELINE_IID && echo $VERSION_CODE" - - "export VERSION_SHA=`echo ${CI_COMMIT_SHA:0:8}` && echo $VERSION_SHA" + - export VERSION_CODE="$CI_PIPELINE_IID" && echo "$VERSION_CODE" + - export VERSION_SHA="${CI_COMMIT_SHA:0:8}" && echo "$VERSION_SHA" after_script: - rm -f android-signing-keystore.jks || true artifacts: diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index a50e722f18a..6354db38f58 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -18,7 +18,7 @@ cache: - .pub-cache/global_packages before_script: - - export PATH="$PATH":"~/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" - pub get --no-precompile test: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index d176ce19299..a5c261e367a 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: image: "cirrusci/flutter:1.22.5" before_script: - pub global activate dart_code_metrics - - export PATH="$PATH":"$HOME/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" script: - metrics lib -r codeclimate > gl-code-quality-report.json artifacts: @@ -20,7 +20,7 @@ test: image: "cirrusci/flutter:1.22.5" before_script: - pub global activate junitreport - - export PATH="$PATH":"$HOME/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" script: - flutter test --machine --coverage | tojunit -o report.xml - lcov --summary coverage/lcov.info diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index b5dd0005013..19e4ffdbe1e 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -16,9 +16,9 @@ variables: # repository in /go/src/gitlab.com/namespace/project # Thus, making a symbolic link corrects this. before_script: - - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) - - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME - - cd $GOPATH/src/$REPO_NAME + - mkdir -p "$GOPATH/src/$(dirname $REPO_NAME)" + - ln -svf "$CI_PROJECT_DIR" "$GOPATH/src/$REPO_NAME" + - cd "$GOPATH/src/$REPO_NAME" stages: - test diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 76f0c9f8427..08dc10d34b7 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -17,7 +17,8 @@ variables: GRADLE_OPTS: "-Dorg.gradle.daemon=false" before_script: - - export GRADLE_USER_HOME=`pwd`/.gradle + - GRADLE_USER_HOME="$(pwd)/.gradle" + - export GRADLE_USER_HOME build: stage: build diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 3c514d7b0c6..7e59354c4a1 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -23,8 +23,8 @@ variables: before_script: - apt-get update -qq && apt-get install -y -qq unzip - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > /root/.sdkman/etc/config - - source /root/.sdkman/bin/sdkman-init.sh + - echo sdkman_auto_answer=true > ~/.sdkman/etc/config + - source ~/.sdkman/bin/sdkman-init.sh - sdk install gradle $GRADLE_VERSION < /dev/null - sdk use gradle $GRADLE_VERSION # As it's not a good idea to version gradle.properties feel free to add your @@ -36,7 +36,7 @@ before_script: # Be aware that if you are using Angular profile, # Bower cannot be run as root if you don't allow it before. # Feel free to remove next line if you are not using Bower - - echo {\"allow_root\":true} > /root/.bowerrc + - echo '{"allow_root":true}' > ~/.bowerrc # This build job does the full grails pipeline # (compile, test, integrationTest, war, assemble). diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 99fd9870b1d..d1018f1e769 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,7 +2,7 @@ browser_performance: stage: performance - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,19 +10,21 @@ browser_performance: SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi - - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - CI_ENVIRONMENT_URL="$(cat environment_url.txt)" + - export CI_ENVIRONMENT_URL - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 99fd9870b1d..bb7e020b159 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -2,7 +2,7 @@ browser_performance: stage: performance - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,11 +10,12 @@ browser_performance: SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi @@ -22,7 +23,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index d5ca93a0a3b..f3d2e293c86 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.5.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' build: stage: build @@ -7,7 +7,7 @@ build: variables: DOCKER_TLS_CERTDIR: '' services: - - name: 'docker:20.10.6-dind' + - name: 'docker:20.10.12-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index d5ca93a0a3b..f3d2e293c86 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.5.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' build: stage: build @@ -7,7 +7,7 @@ build: variables: DOCKER_TLS_CERTDIR: '' services: - - name: 'docker:20.10.6-dind' + - name: 'docker:20.10.12-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6942631a97f..6a95d042842 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -1,9 +1,10 @@ code_quality: stage: test - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" @@ -13,7 +14,7 @@ code_quality: - export SOURCE_CODE=$PWD - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 28ac627f103..cc204207f84 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.17.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 65c9232f3b9..1a99db67441 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 075e13e87f0..bc4f2099d94 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.18.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e9c5d970c21..ce584091eab 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.18.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index fc51f5adb3c..89a44eddefd 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. LICENSE_MANAGEMENT_VERSION: 3 diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 8e34388893a..eea1c397108 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -1,6 +1,6 @@ load_performance: stage: performance - image: docker:19.03.11 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,11 +10,12 @@ load_performance: K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: - - docker:19.03.11-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index fa7f6ffa2b7..5ddfb2a54be 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" iac-sast: diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 25d20563010..8cc9ea0200c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 4e4f96bc7c7..0ef6f63bb94 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -5,7 +5,7 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECRETS_ANALYZER_VERSION: "3" SECRET_DETECTION_EXCLUDED_PATHS: "" @@ -30,24 +30,43 @@ secret_detection: - if: $CI_COMMIT_BRANCH script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + # Historic scan - | - # we don't need the whole history when excluding in the next `git fetch` line, - # so git depth=1 - git fetch origin --depth=1 $CI_DEFAULT_BRANCH - # shallow clone $CI_COMMIT_REF_NAME to get commits associated with MR or push - git fetch --shallow-exclude=${CI_DEFAULT_BRANCH} origin $CI_COMMIT_REF_NAME - # determine what commits we need to scan using "git log A..B" - git log --no-merges --pretty=format:"%H" refs/remotes/origin/${CI_DEFAULT_BRANCH}..refs/remotes/origin/${CI_COMMIT_REF_NAME} >${CI_COMMIT_SHA}_commit_list.txt - - # we need to extend the git fetch depth to the number of commits + 2 for the following reasons: - # because busybox wc only counts \n and there is no trailing \n (+1) - # include the parent commit of the base commit in this MR/Push event. This is needed because - # `git diff -p` needs something to compare changes in that commit against (+1) - git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 2)) origin $CI_COMMIT_REF_NAME + if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ] + then + echo "historic scan" + git fetch --unshallow origin $CI_COMMIT_REF_NAME + /analyzer run + exit + fi + # Default branch scan + - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit; fi + # Push event + - | + if [ "$CI_COMMIT_BEFORE_SHA" == "0000000000000000000000000000000000000000" ]; + then + # first commit on a new branch + echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt + git fetch --depth=2 origin $CI_COMMIT_REF_NAME + else + # determine commit range so that we can fetch the appropriate depth + # check the exit code to determine if we need to limit the commit_list.txt to CI_COMMIT_SHA. + if ! git log --pretty=format:"%H" ${CI_COMMIT_BEFORE_SHA}..${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt; + then + echo "unable to determine commit range, limiting to ${CI_COMMIT_SHA}" + echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt + else + # append newline to to list since `git log` does not end with a + # newline, this is to keep the log messages consistent + echo >> ${CI_COMMIT_SHA}_commit_list.txt + fi - # +1 because busybox wc only counts \n and there is no trailing \n - echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) commits" + # we need to extend the git fetch depth to the number of commits + 1 for the following reasons: + # to include the parent commit of the base commit in this MR/Push event. This is needed because + # `git diff -p` needs something to compare changes in that commit against + git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) origin $CI_COMMIT_REF_NAME + fi + echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt))) commits for a push event" export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index d32444833fb..85f90984045 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -8,7 +8,7 @@ pages: stage: deploy script: - mkdir .public - - cp -r * .public + - cp -r ./* .public - rm -rf public - mv .public public artifacts: diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 4917abf6ae9..6ed5e05ed4c 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -47,7 +47,8 @@ run: pages: script: - pip install sphinx sphinx-rtd-theme - - cd doc ; make html + - cd doc + - make html - mv build/html/ ../public/ artifacts: paths: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 1660a9250e3..33c0928db6f 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -52,7 +52,7 @@ rails: # This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk # are supported too: https://github.com/travis-ci/dpl deploy: - type: deploy + stage: deploy environment: production script: - gem install dpl diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 009061ce844..aff8b6cb7fa 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: FUZZAPI_VERSION: "1" - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} apifuzzer_fuzz: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 01041f4f056..bd8ba71effe 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: FUZZAPI_VERSION: "1" - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index a2933085d4e..d82f9f06f8d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 57f1993921d..0e0afa489a3 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" DAST_API_IMAGE: api-fuzzing diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 7ffec7d2e6b..3f9c87b7abf 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -5,7 +5,7 @@ stages: - dast variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" DAST_API_VERSION: "1" DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 3e7ab9b5c3b..998425aa141 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 0ecbe5e14b8..e8e7fe62e70 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 3d07674c377..c755211ec11 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 82c7bfd0620..a6fd070ec34 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -14,8 +14,11 @@ # Docs: https://docs.gitlab.com/ee/topics/airgap/ variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, license-finder, dast, dast-runner-validation, api-fuzzing @@ -40,7 +43,7 @@ variables: script: - docker info - env - - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/analyzers/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi + - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"${SECURE_ANALYZERS_PREFIX}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi - docker pull --quiet ${SECURE_BINARIES_IMAGE} - mkdir -p output/$(dirname ${CI_JOB_NAME}) - | diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index e696c75253e..84a962e1541 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -24,19 +24,19 @@ cache: .init: &init stage: init script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform init .validate: &validate stage: validate script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform validate .build: &build stage: build script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json artifacts: @@ -48,7 +48,7 @@ cache: .deploy: &deploy stage: deploy script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform apply when: manual only: @@ -58,6 +58,6 @@ cache: .destroy: &destroy stage: cleanup script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform destroy when: manual diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 8f4a836441d..5ea2bc07ffa 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -14,7 +14,8 @@ stages: a11y: stage: accessibility image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 - script: /gitlab-accessibility.sh $a11y_urls + script: + - /gitlab-accessibility.sh "$a11y_urls" allow_failure: true artifacts: when: always diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index e0df9799917..2349c37c130 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -25,7 +25,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index ad24ebae8d4..73ab5fcbe44 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -25,7 +25,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb index 7f43d91e6d7..eaa9be9dd15 100644 --- a/lib/gitlab/ci/trace/remote_checksum.rb +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -23,6 +23,7 @@ module Gitlab private attr_reader :trace_artifact + delegate :aws?, :google?, to: :object_store_config, prefix: :provider def fetch_md5_checksum diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 9ef6e7f5fa9..bfcf67693e7 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -10,6 +10,7 @@ module Gitlab @pipeline = pipeline @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) + @group_variables_builder = Builder::Group.new(project.group) end def scoped_variables(job, environment:, dependencies:) @@ -18,8 +19,7 @@ module Gitlab variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner - variables.concat(kubernetes_variables(job)) - variables.concat(deployment_variables(environment: environment, job: job)) + variables.concat(kubernetes_variables(environment: environment, job: job)) variables.concat(job.yaml_variables) variables.concat(user_variables(job.user)) variables.concat(job.dependency_variables) if dependencies @@ -32,11 +32,15 @@ module Gitlab end end - def kubernetes_variables(job) + def kubernetes_variables(environment:, job:) ::Gitlab::Ci::Variables::Collection.new.tap do |collection| - # Should get merged with the cluster kubeconfig in deployment_variables, see - # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 + # NOTE: deployment_variables will be removed as part of cleanup for + # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 + # Until then, we need to make both the old and the new KUBECONFIG contexts available + collection.concat(deployment_variables(environment: environment, job: job)) template = ::Ci::GenerateKubeconfigService.new(job).execute + kubeconfig_yaml = collection['KUBECONFIG']&.value + template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? if template.valid? collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) @@ -72,9 +76,13 @@ module Gitlab end def secret_group_variables(environment:, ref:) - return [] unless project.group + if memoize_secret_variables? + memoized_secret_group_variables(environment: environment) + else + return [] unless project.group - project.group.ci_variables_for(ref, project, environment: environment) + project.group.ci_variables_for(ref, project, environment: environment) + end end def secret_project_variables(environment:, ref:) @@ -90,6 +98,8 @@ module Gitlab attr_reader :pipeline attr_reader :instance_variables_builder attr_reader :project_variables_builder + attr_reader :group_variables_builder + delegate :project, to: :pipeline def predefined_variables(job) @@ -119,6 +129,15 @@ module Gitlab end end + def memoized_secret_group_variables(environment:) + strong_memoize_with(:secret_group_variables, environment) do + group_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) + end + end + def ci_node_total_value(job) parallel = job.options&.dig(:parallel) parallel = parallel.dig(:total) if parallel.is_a?(Hash) diff --git a/lib/gitlab/ci/variables/builder/group.rb b/lib/gitlab/ci/variables/builder/group.rb new file mode 100644 index 00000000000..3f3e04038df --- /dev/null +++ b/lib/gitlab/ci/variables/builder/group.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Group + include Gitlab::Utils::StrongMemoize + + def initialize(group) + @group = group + end + + def secret_variables(environment:, protected_ref: false) + return [] unless group + + variables = base_scope + variables = variables.unprotected unless protected_ref + variables = variables.for_environment(environment) + variables = variables.group_by(&:group_id) + variables = list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact + Gitlab::Ci::Variables::Collection.new(variables) + end + + private + + attr_reader :group + + def base_scope + strong_memoize(:base_scope) do + ::Ci::GroupVariable.for_groups(list_of_ids) + end + end + + def list_of_ids + strong_memoize(:list_of_ids) do + if group.root_ancestor.use_traversal_ids? + [group] + group.ancestors(hierarchy_order: :asc) + else + [group] + group.ancestors + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 553508c8638..15ebd506055 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -45,7 +45,7 @@ module Gitlab validate_job!(name, job) end - YamlProcessor::Dag.check_circular_dependencies!(@jobs) + check_circular_dependencies end def validate_job!(name, job) @@ -146,6 +146,17 @@ module Gitlab end end + def check_circular_dependencies + jobs = @jobs.values.to_h do |job| + name = job[:name].to_s + needs = job.dig(:needs, :job).to_a + + [name, needs.map { |need| need[:name].to_s }] + end + + Dag.check_circular_dependencies!(jobs) + end + def error!(message) raise ValidationError, message end diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb index 8ab9573dd20..4a122c73e80 100644 --- a/lib/gitlab/ci/yaml_processor/dag.rb +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -7,28 +7,22 @@ module Gitlab class Dag include TSort - MissingNodeError = Class.new(StandardError) - def initialize(nodes) @nodes = nodes end - def self.check_circular_dependencies!(jobs) - nodes = jobs.values.to_h do |job| - name = job[:name].to_s - needs = job.dig(:needs, :job).to_a - - [name, needs.map { |need| need[:name].to_s }] - end + def self.order(jobs) + new(jobs).tsort + end - new(nodes).tsort + def self.check_circular_dependencies!(jobs) + new(jobs).tsort rescue TSort::Cyclic raise ValidationError, 'The pipeline has circular dependencies' - rescue MissingNodeError end def tsort_each_child(node, &block) - raise MissingNodeError, "node #{node} is missing" unless @nodes[node] + return unless @nodes[node] @nodes[node].each(&block) end |