diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 15:40:28 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 15:40:28 +0000 |
| commit | b595cb0c1dec83de5bdee18284abe86614bed33b (patch) | |
| tree | 8c3d4540f193c5ff98019352f554e921b3a41a72 /lib/gitlab/ci | |
| parent | 2f9104a328fc8a4bddeaa4627b595166d24671d0 (diff) | |
| download | gitlab-ce-b595cb0c1dec83de5bdee18284abe86614bed33b.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-2-stable-eev15.2.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
58 files changed, 600 insertions, 274 deletions
diff --git a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb deleted file mode 100644 index 848208c5cdd..00000000000 --- a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Artifacts - class ExpireInParser - def self.validate_duration(value) - new(value).validate_duration - end - - def initialize(value) - @value = value - end - - def validate_duration - return true if never? - - cached_parse - end - - def seconds_from_now - parse&.seconds&.from_now - end - - private - - attr_reader :value - - def cached_parse - return validation_cache[value] if validation_cache.key?(value) - - validation_cache[value] = safe_parse - end - - def safe_parse - parse - rescue ChronicDuration::DurationParseError - false - end - - def parse - return if never? - - ChronicDuration.parse(value) - end - - def validation_cache - Gitlab::SafeRequestStore[:ci_expire_in_parser_cache] ||= {} - end - - def never? - value.to_s.casecmp('never') == 0 - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/duration_parser.rb b/lib/gitlab/ci/build/duration_parser.rb new file mode 100644 index 00000000000..9385dccd5f3 --- /dev/null +++ b/lib/gitlab/ci/build/duration_parser.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class DurationParser + def self.validate_duration(value) + new(value).validate_duration + end + + def initialize(value) + @value = value + end + + def validate_duration + return true if never? + + cached_parse + end + + def seconds_from_now + parse&.seconds&.from_now + end + + private + + attr_reader :value + + def cached_parse + return validation_cache[value] if validation_cache.key?(value) + + validation_cache[value] = safe_parse + end + + def safe_parse + parse + rescue ChronicDuration::DurationParseError + false + end + + def parse + return if never? + + ChronicDuration.parse(value) + end + + def validation_cache + Gitlab::SafeRequestStore[:ci_expire_in_parser_cache] ||= {} + end + + def never? + value.to_s.casecmp('never') == 0 + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 4c5f02b4f7b..1bcd87c9d93 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -4,8 +4,10 @@ module Gitlab module Ci module Build class Rules::Rule::Clause::Changes < Rules::Rule::Clause + include Gitlab::Utils::StrongMemoize + def initialize(globs) - @globs = Array(globs) + @globs = globs end def satisfied_by?(pipeline, context) @@ -19,13 +21,25 @@ module Gitlab end end + private + def expand_globs(context) - return @globs unless context + return paths unless context - @globs.map do |glob| + paths.map do |glob| ExpandVariables.expand_existing(glob, -> { context.variables_hash }) end end + + def paths + strong_memoize(:paths) do + if @globs.is_a?(Array) + @globs + else + Array(@globs[:paths]) + end + end + end end end end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 56eeb5eeb06..3b0cbc6b69e 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -42,7 +42,7 @@ module Gitlab inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure ' \ 'or always' } - validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::Artifacts::ExpireInParser } + validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser } end end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index bc39abfe977..96ba3553b46 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -54,7 +54,7 @@ module Gitlab validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true - validates :auto_stop_in, duration: true, allow_nil: true + validates :auto_stop_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser }, allow_nil: true end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 79443f69b03..96ac959a3f4 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -48,7 +48,7 @@ module Gitlab { name: @config[:name], entrypoint: @config[:entrypoint], - ports: ports_value, + ports: (ports_value if ports_defined?), pull_policy: (ci_docker_image_pull_policy_enabled? ? pull_policy_value : nil) }.compact else diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 46afedbcc3a..78794f524f4 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -71,9 +71,9 @@ module Gitlab end def compose!(deps = nil) - super do - has_workflow_rules = deps&.workflow_entry&.has_rules? + has_workflow_rules = deps&.workflow_entry&.has_rules? + super do # If workflow:rules: or rules: are used # they are considered not compatible # with `only/except` defaults @@ -86,12 +86,10 @@ module Gitlab @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables end - unless has_workflow_rules - validate_against_warnings - end - yield if block_given? end + + validate_against_warnings unless has_workflow_rules end def validate_against_warnings diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index 53e52981471..91be1bb3ee4 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -13,7 +13,7 @@ module Gitlab end def value - [@config].flatten + [super].flatten end def composable_class diff --git a/lib/gitlab/ci/config/entry/rules/rule/changes.rb b/lib/gitlab/ci/config/entry/rules/rule/changes.rb index be57e089f34..a56b928450a 100644 --- a/lib/gitlab/ci/config/entry/rules/rule/changes.rb +++ b/lib/gitlab/ci/config/entry/rules/rule/changes.rb @@ -6,13 +6,51 @@ module Gitlab module Entry class Rules class Rule - class Changes < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable + class Changes < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleChanges, if: -> (config) { config.is_a?(Array) } + strategy :ComplexChanges, if: -> (config) { config.is_a?(Hash) } - validations do - validates :config, - array_of_strings: true, - length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + class SimpleChanges < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, + array_of_strings: true, + length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + end + + def value + { + paths: config + }.compact + end + end + + class ComplexChanges < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[paths].freeze + REQUIRED_KEYS = %i[paths].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_KEYS + + with_options allow_nil: false do + validates :paths, + array_of_strings: true, + length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + end + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be an array or a hash"] + end end end end diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index f27dca4986e..1a35f7de6cf 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -15,11 +15,13 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze + ALLOWED_KEYS = %i[name entrypoint command alias ports variables pull_policy].freeze + LEGACY_ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze validations do validates :config, hash_or_string: true - validates :config, allowed_keys: ALLOWED_KEYS + validates :config, allowed_keys: ALLOWED_KEYS, if: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: LEGACY_ALLOWED_KEYS, unless: :ci_docker_image_pull_policy_enabled? validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true @@ -32,11 +34,14 @@ module Gitlab entry :ports, Entry::Ports, description: 'Ports used to expose the service' + entry :pull_policy, Entry::PullPolicy, + description: 'Pull policy for the service' + entry :variables, ::Gitlab::Ci::Config::Entry::Variables, description: 'Environment variables available for this service.', inherit: false - attributes :ports + attributes :ports, :pull_policy, :variables def alias value[:alias] @@ -55,16 +60,25 @@ module Gitlab end def value - return { name: @config } if string? - return @config if hash? - - {} + if string? + { name: @config } + elsif hash? + @config.merge( + pull_policy: (pull_policy_value if ci_docker_image_pull_policy_enabled?) + ).compact + else + {} + end end def with_image_ports? opt(:with_image_ports) end + def ci_docker_image_pull_policy_enabled? + ::Feature.enabled?(:ci_docker_image_pull_policy) + end + def skip_config_hash_validation? true end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 2def565bc19..ec628399785 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,14 +9,20 @@ module Gitlab TimeoutError = Class.new(StandardError) + MAX_INCLUDES = 100 + TRIAL_MAX_INCLUDES = 250 + include ::Gitlab::Utils::StrongMemoize attr_reader :project, :sha, :user, :parent_pipeline, :variables - attr_reader :expandset, :execution_deadline, :logger + attr_reader :expandset, :execution_deadline, :logger, :max_includes delegate :instrument, to: :logger - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, logger: nil) + def initialize( + project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, + logger: nil + ) @project = project @sha = sha @user = user @@ -25,7 +31,7 @@ module Gitlab @expandset = Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - + @max_includes = Feature.enabled?(:ci_increase_includes_to_250, project) ? TRIAL_MAX_INCLUDES : MAX_INCLUDES yield self if block_given? end @@ -52,6 +58,7 @@ module Gitlab ctx.expandset = expandset ctx.execution_deadline = execution_deadline ctx.logger = logger + ctx.max_includes = max_includes end end @@ -86,7 +93,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline, :logger + attr_writer :expandset, :execution_deadline, :logger, :max_includes private diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index b7fef081269..89418bd6a21 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -13,7 +13,7 @@ module Gitlab def initialize(params, context) @location = params[:file] - @project_name = params[:project] + @project_name = get_project_name(params[:project]) @ref_name = params[:ref] || 'HEAD' super @@ -122,6 +122,16 @@ module Gitlab ) end end + + # TODO: To be removed after we deprecate usage of array in `project` keyword. + # https://gitlab.com/gitlab-org/gitlab/-/issues/365975 + def get_project_name(project_name) + if project_name.is_a?(Array) + project_name.first + else + project_name + end + end end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index c1250c82750..2a1060a6059 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,8 +7,6 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize - MAX_INCLUDES = 100 - FILE_CLASSES = [ External::File::Remote, External::File::Template, @@ -134,8 +132,8 @@ module Gitlab end def verify_max_includes! - if expandset.count >= MAX_INCLUDES - raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + if expandset.count >= context.max_includes + raise TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 19678def666..c294291e538 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -64,7 +64,8 @@ module Gitlab if environment.present? fields.merge!( environment: environment.name, - environment_protected: environment_protected?.to_s + environment_protected: environment_protected?.to_s, + deployment_tier: build.environment_deployment_tier || environment.tier ) end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 71dfc1a676c..207b4b5ff8b 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -11,10 +11,10 @@ module Gitlab def perform! logger.instrument_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do - with_bulk_insert_tags do + ::Ci::BulkInsertableTags.with_bulk_insert_tags do pipeline.transaction do pipeline.save! - CommitStatus.bulk_insert_tags!(statuses) + Gitlab::Ci::Tags::BulkInsert.bulk_insert_tags!(statuses) end end end @@ -29,14 +29,6 @@ module Gitlab private - def with_bulk_insert_tags - previous = Thread.current['ci_bulk_insert_tags'] - Thread.current['ci_bulk_insert_tags'] = true - yield - ensure - Thread.current['ci_bulk_insert_tags'] = previous - end - def statuses strong_memoize(:statuses) do pipeline diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 33b9ac9b641..c3e0f043b44 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -42,6 +42,15 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_age_histogram + name = :gitlab_ci_pipeline_age_minutes + comment = 'Pipeline age histogram' + buckets = [5, 30, 120, 720, 1440, 7200, 21600, 43200, 86400, 172800, 518400, 1036800] + # 5m 30m 2h 12h 24h 5d 15d 30d 60d 180d 360d 2y + + ::Gitlab::Metrics.histogram(name, comment, {}, buckets) + end + def self.active_jobs_histogram name = :gitlab_ci_active_jobs comment = 'Total amount of active jobs' diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7d8303214a5..5cee73238ca 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -250,11 +250,7 @@ module Gitlab end def running_jobs_relation(job) - if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data) - ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) - else - job.project.builds.running.where(runner: ::Ci::Runner.instance_type) - end + ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/ci/reports/coverage_report_generator.rb b/lib/gitlab/ci/reports/coverage_report_generator.rb index fd73ed6fd25..76992a48b0a 100644 --- a/lib/gitlab/ci/reports/coverage_report_generator.rb +++ b/lib/gitlab/ci/reports/coverage_report_generator.rb @@ -35,17 +35,7 @@ module Gitlab private def report_builds - if child_pipeline_feature_enabled? - @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) - else - @pipeline.latest_report_builds(::Ci::JobArtifact.coverage_reports) - end - end - - def child_pipeline_feature_enabled? - strong_memoize(:feature_enabled) do - Feature.enabled?(:ci_child_pipeline_coverage_reports, @pipeline.project) - end + @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) end end end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_report.rb index a5a630642e5..4fc10dd736e 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_report.rb @@ -3,7 +3,7 @@ module Gitlab module Ci module Reports - class TestReports + class TestReport attr_reader :test_suites def initialize diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index c6f17f0764f..497831ae5a7 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :base_reports, :head_reports def initialize(base_reports, head_reports) - @base_reports = base_reports || TestReports.new + @base_reports = base_reports || TestReport.new @head_reports = head_reports end diff --git a/lib/gitlab/ci/runner/metrics.rb b/lib/gitlab/ci/runner/metrics.rb new file mode 100644 index 00000000000..8df126decff --- /dev/null +++ b/lib/gitlab/ci/runner/metrics.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + class Metrics + extend Gitlab::Utils::StrongMemoize + + def increment_runner_authentication_success_counter(runner_type: 'unknown_type') + raise ArgumentError, "unknown runner type: #{runner_type}" unless + ::Ci::Runner.runner_types.include? runner_type + + self.class.runner_authentication_success_counter.increment(runner_type: runner_type) + end + + def increment_runner_authentication_failure_counter + self.class.runner_authentication_failure_counter.increment + end + + def self.runner_authentication_success_counter + strong_memoize(:runner_authentication_success) do + name = :gitlab_ci_runner_authentication_success_total + comment = 'Runner authentication success' + labels = { runner_type: nil } + + ::Gitlab::Metrics.counter(name, comment, labels) + end + end + + def self.runner_authentication_failure_counter + strong_memoize(:runner_authentication_failure) do + name = :gitlab_ci_runner_authentication_failure_total + comment = 'Runner authentication failure' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb index 944c24ca128..8773ecbf09e 100644 --- a/lib/gitlab/ci/runner_releases.rb +++ b/lib/gitlab/ci/runner_releases.rb @@ -6,48 +6,83 @@ module Gitlab include Singleton RELEASES_VALIDITY_PERIOD = 1.day - RELEASES_VALIDITY_AFTER_ERROR_PERIOD = 5.seconds INITIAL_BACKOFF = 5.seconds MAX_BACKOFF = 1.hour BACKOFF_GROWTH_FACTOR = 2.0 def initialize - reset! + reset_backoff! end # Returns a sorted list of the publicly available GitLab Runner releases # def releases - return @releases unless Time.now.utc >= @expire_time + return if backoff_active? + + Rails.cache.fetch( + cache_key, + skip_nil: true, + expires_in: RELEASES_VALIDITY_PERIOD, + race_condition_ttl: 10.seconds + ) do + response = Gitlab::HTTP.try_get(runner_releases_url) + @releases_by_minor = nil + + unless response&.success? + @backoff_expire_time = next_backoff.from_now + break nil + end + + reset_backoff! + extract_releases(response) + end + end + + # Returns a hash with the latest runner version per minor release + # + def releases_by_minor + return unless releases - @releases = fetch_new_releases + @releases_by_minor ||= releases.group_by(&:without_patch).transform_values(&:max) end - def reset! - @expire_time = Time.now.utc - @releases = nil + def reset_backoff! + @backoff_expire_time = nil @backoff_count = 0 end - public_class_method :instance - private - def fetch_new_releases - response = Gitlab::HTTP.try_get(::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url) + def runner_releases_url + @runner_releases_url ||= ::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url + end - releases = response.success? ? extract_releases(response) : nil - ensure - @expire_time = (releases ? RELEASES_VALIDITY_PERIOD : next_backoff).from_now + def cache_key + runner_releases_url + end + + def backoff_active? + return false unless @backoff_expire_time + + Time.now.utc < @backoff_expire_time end def extract_releases(response) - response.parsed_response.map { |release| parse_runner_release(release) }.sort! + return unless response.parsed_response.is_a?(Array) + + releases = response.parsed_response + .map { |release| parse_runner_release(release) } + .select(&:valid?) + .sort! + + return if releases.empty? && response.parsed_response.present? + + releases end def parse_runner_release(release) - ::Gitlab::VersionInfo.parse(release['name'].delete_prefix('v')) + ::Gitlab::VersionInfo.parse(release['name'], parse_suffix: true) end def next_backoff diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index 0808290fe5b..10a89bb15d4 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -5,76 +5,71 @@ module Gitlab class RunnerUpgradeCheck include Singleton - STATUSES = { - invalid: 'Runner version is not valid.', - not_available: 'Upgrade is not available for the runner.', - available: 'Upgrade is available for the runner.', - recommended: 'Upgrade is available and recommended for the runner.' - }.freeze - - def initialize - reset! - end - def check_runner_upgrade_status(runner_version) - return :invalid unless runner_version + runner_version = ::Gitlab::VersionInfo.parse(runner_version, parse_suffix: true) - releases = RunnerReleases.instance.releases - orig_runner_version = runner_version - runner_version = ::Gitlab::VersionInfo.parse(runner_version) unless runner_version.is_a?(::Gitlab::VersionInfo) + return { invalid_version: runner_version } unless runner_version.valid? + return { error: runner_version } unless runner_releases_store.releases - raise ArgumentError, "'#{orig_runner_version}' is not a valid version" unless runner_version.valid? + # Recommend update if outside of backport window + recommended_version = recommendation_if_outside_backport_window(runner_version) + return { recommended: recommended_version } if recommended_version - gitlab_minor_version = version_without_patch(@gitlab_version) + # Recommend patch update if there's a newer release in a same minor branch as runner + recommended_version = recommended_runner_release_update(runner_version) + return { recommended: recommended_version } if recommended_version - available_releases = releases - .reject { |release| release.major > @gitlab_version.major } - .reject do |release| - release_minor_version = version_without_patch(release) + # Consider update if there's a newer release within the currently deployed GitLab version + available_version = available_runner_release(runner_version) + return { available: available_version } if available_version - # Do not reject a patch update, even if the runner is ahead of the instance version - next false if version_without_patch(runner_version) == release_minor_version + { not_available: runner_version } + end - release_minor_version > gitlab_minor_version - end + private - return :recommended if available_releases.any? { |available_rel| patch_update?(available_rel, runner_version) } - return :recommended if outside_backport_window?(runner_version, releases) - return :available if available_releases.any? { |available_rel| available_rel > runner_version } + def recommended_runner_release_update(runner_version) + recommended_release = runner_releases_store.releases_by_minor[runner_version.without_patch] + return recommended_release if recommended_release && recommended_release > runner_version - :not_available + # Consider the edge case of pre-release runner versions that get registered, but are never published. + # In this case, suggest the latest compatible runner version + latest_release = runner_releases_store.releases_by_minor.values.select { |v| v < gitlab_version }.max + latest_release if latest_release && latest_release > runner_version end - def reset! - @gitlab_version = ::Gitlab::VersionInfo.parse(::Gitlab::VERSION) + def available_runner_release(runner_version) + available_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] + available_release if available_release && available_release > runner_version end - public_class_method :instance - - private - - def patch_update?(available_release, runner_version) - # https://docs.gitlab.com/ee/policy/maintenance.html#patch-releases - available_release.major == runner_version.major && - available_release.minor == runner_version.minor && - available_release.patch > runner_version.patch + def gitlab_version + @gitlab_version ||= ::Gitlab::VersionInfo.parse(::Gitlab::VERSION, parse_suffix: true) end - def outside_backport_window?(runner_version, releases) - return false if runner_version >= releases.last # return early if runner version is too new - - latest_minor_releases = releases.map { |r| version_without_patch(r) }.uniq { |v| v.to_s } - latest_version_position = latest_minor_releases.count - 1 - runner_version_position = latest_minor_releases.index(version_without_patch(runner_version)) - - return true if runner_version_position.nil? # consider outside if version is too old - - # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases - latest_version_position - runner_version_position > 2 + def runner_releases_store + RunnerReleases.instance end - def version_without_patch(version) - ::Gitlab::VersionInfo.new(version.major, version.minor, 0) + def recommendation_if_outside_backport_window(runner_version) + return if runner_releases_store.releases.empty? + return if runner_version >= runner_releases_store.releases.last # return early if runner version is too new + + minor_releases_with_index = runner_releases_store.releases_by_minor.keys.each_with_index.to_h + runner_minor_version_index = minor_releases_with_index[runner_version.without_patch] + if runner_minor_version_index + # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases + outside_window = minor_releases_with_index.count - runner_minor_version_index > 3 + + if outside_window + recommended_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] + + recommended_release if recommended_release && recommended_release > runner_version + end + else + # If unknown runner version, then recommend the latest version for the GitLab instance + recommended_runner_release_update(gitlab_version) + end end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 1a074c1af53..5d60aa8f540 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -20,10 +20,13 @@ module Gitlab scheduler_failure: 'scheduler failure', data_integrity_failure: 'data integrity failure', forward_deployment_failure: 'forward deployment failure', + protected_environment_failure: 'protected environment failure', pipeline_loop_detected: 'job would create infinitely looping pipelines', invalid_bridge_trigger: 'downstream pipeline trigger definition is invalid', downstream_bridge_project_not_found: 'downstream project could not be found', + upstream_bridge_project_not_found: 'upstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', + insufficient_upstream_permissions: 'no permissions to read upstream project', bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', @@ -75,5 +78,3 @@ module Gitlab end end end - -Gitlab::Ci::Status::Build::Failed.prepend_mod_with('Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 3b2da773102..e854164d377 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,10 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - # - # The parameter `project` is only used for the feature flag check, and will be removed with - # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 - def initialize(all_statuses, with_allow_failure: true, dag: false, project: nil) + def initialize(all_statuses, with_allow_failure: true, dag: false) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -19,7 +16,6 @@ module Gitlab @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag - @project = project consume_all_statuses(all_statuses) end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb index 29f3731a9b4..2e56e47f5b8 100644 --- a/lib/gitlab/ci/tags/bulk_insert.rb +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -9,40 +9,44 @@ module Gitlab TAGGINGS_BATCH_SIZE = 1000 TAGS_BATCH_SIZE = 500 - def initialize(statuses) - @statuses = statuses + def self.bulk_insert_tags!(taggables) + Gitlab::Ci::Tags::BulkInsert.new(taggables).insert! + end + + def initialize(taggables) + @taggables = taggables end def insert! - return false if tag_list_by_status.empty? + return false if tag_list_by_taggable.empty? persist_build_tags! end private - attr_reader :statuses + attr_reader :taggables - def tag_list_by_status - strong_memoize(:tag_list_by_status) do - statuses.each.with_object({}) do |status, acc| - tag_list = status.tag_list + def tag_list_by_taggable + strong_memoize(:tag_list_by_taggable) do + taggables.each.with_object({}) do |taggable, acc| + tag_list = taggable.tag_list next unless tag_list - acc[status] = tag_list + acc[taggable] = tag_list end end end def persist_build_tags! - all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) + all_tags = tag_list_by_taggable.values.flatten.uniq.reject(&:blank?) tag_records_by_name = create_tags(all_tags).index_by(&:name) taggings = build_taggings_attributes(tag_records_by_name) return false if taggings.empty? taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice| - ActsAsTaggableOn::Tagging.insert_all!(taggings) + ActsAsTaggableOn::Tagging.insert_all!(taggings_slice) end true @@ -65,24 +69,24 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def build_taggings_attributes(tag_records_by_name) - taggings = statuses.flat_map do |status| - tag_list = tag_list_by_status[status] + taggings = taggables.flat_map do |taggable| + tag_list = tag_list_by_taggable[taggable] next unless tag_list tags = tag_records_by_name.values_at(*tag_list) - taggings_for(tags, status) + taggings_for(tags, taggable) end taggings.compact! taggings end - def taggings_for(tags, status) + def taggings_for(tags, taggable) tags.map do |tag| { tag_id: tag.id, - taggable_type: CommitStatus.name, - taggable_id: status.id, + taggable_type: taggable.class.base_class.name, + taggable_id: taggable.id, created_at: Time.current, context: 'tags' } diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml index 17e49440784..1ac9c319429 100644 --- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -7,6 +7,7 @@ stages: - build - test - review + - dast - deploy - production - cleanup diff --git a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml index 9f0e9bcc1f2..ee52bc91ab3 100644 --- a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml @@ -30,24 +30,24 @@ before_script: - apt-get --quiet update --yes - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 - # Setup path as ANDROID_SDK_ROOT for moving/exporting the downloaded sdk into it - - export ANDROID_SDK_ROOT="${PWD}/android-home" + # Setup path as ANDROID_HOME for moving/exporting the downloaded sdk into it + - export ANDROID_HOME="${PWD}/android-home" # Create a new directory at specified location - - install -d $ANDROID_SDK_ROOT + - install -d $ANDROID_HOME # Here we are installing androidSDK tools from official source, # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well) # after that unzipping those tools and # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build - - wget --output-document=$ANDROID_SDK_ROOT/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip - # move to the archive at ANDROID_SDK_ROOT - - pushd $ANDROID_SDK_ROOT + - wget --output-document=$ANDROID_HOME/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip + # move to the archive at ANDROID_HOME + - pushd $ANDROID_HOME - unzip -d cmdline-tools cmdline-tools.zip - pushd cmdline-tools # since commandline tools version 7583922 the root folder is named "cmdline-tools" so we rename it if necessary - mv cmdline-tools tools || true - popd - popd - - export PATH=$PATH:${ANDROID_SDK_ROOT}/cmdline-tools/tools/bin/ + - export PATH=$PATH:${ANDROID_HOME}/cmdline-tools/tools/bin/ # Nothing fancy here, just checking sdkManager version - sdkmanager --version diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index f39a84bceec..004c2897b60 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index c078c99f352..3096af1b173 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index f166da9bdd6..a64f87193a9 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -1,14 +1,17 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml - # This template uses Test Kitchen with the kitchen-dokken driver to # perform functional testing. Doing so requires that your runner be a # Docker runner configured for privileged mode. Please see # https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode # for help configuring your runner properly, or, if you want to switch # to a different driver, see http://kitchen.ci/docs/drivers +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml image: "chef/chefdk" services: diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 0f9e28c9a8e..4fe37ceaeaa 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 8886929646d..68b55b782cd 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index 6354db38f58..35401e62fe2 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 1eb920c7747..83ddce936e6 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index a5c261e367a..021662ab416 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 21a599fc78d..464b81965f2 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - # This is a sample GitLab CI/CD configuration file that should run without any modifications. # It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, # it uses echo commands to simulate the pipeline execution. @@ -11,6 +6,14 @@ # Stages run in sequential order, but jobs within stages run in parallel. # # For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml stages: # List of stages for jobs, and their order of execution - build diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index bd8e1020c4e..603aede4d46 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 7e59354c4a1..03c8941169f 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml - # This template uses the java:8 docker image because there isn't any # official Grails image at this moment # @@ -12,6 +7,14 @@ # Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...) # Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...) # If you use Angular profile, this yml it's prepared to work with it +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml image: java:8 diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6a95d042842..86e3ace84c5 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.26" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.29" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 8f1124373c4..b41e92e3a56 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -4,6 +4,14 @@ variables: .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" +.common_rules: &common_rules + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + when: never + - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + when: never + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + when: never + dast_environment_deploy: extends: .dast-auto-deploy stage: review @@ -23,12 +31,7 @@ dast_environment_deploy: artifacts: paths: [environment_url.txt] rules: - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - when: never - - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given - when: never + - *common_rules - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ @@ -47,13 +50,53 @@ stop_dast_environment: action: stop needs: ["dast"] rules: - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - when: never - - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given - when: never + - *common_rules - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ when: always + +.ecs_image: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + +.ecs_rules: &ecs_rules + - if: $AUTO_DEVOPS_PLATFORM_TARGET != "ECS" + when: never + - if: $CI_KUBERNETES_ACTIVE || $KUBECONFIG + when: never + +dast_ecs_environment_deploy: + extends: .ecs_image + stage: review + script: + - ecs update-task-definition + - echo "http://$(ecs get-task-hostname)" > environment_url.txt + environment: + name: dast-default + on_stop: stop_dast_ecs_environment + artifacts: + paths: + - environment_url.txt + rules: + - *common_rules + - *ecs_rules + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + +stop_dast_ecs_environment: + extends: .ecs_image + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - ecs stop-task + allow_failure: true + environment: + name: dast-default + action: stop + needs: + - dast + rules: + - *common_rules + - *ecs_rules + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + when: always diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index b95b36fd555..a9d9c400a34 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -46,10 +46,10 @@ dependency_scanning: script: - /analyzer run -.cyclone-dx-reports: +.cyclonedx-reports: artifacts: paths: - - "**/cyclonedx-*.json" + - "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: @@ -66,7 +66,7 @@ dependency_scanning: gemnasium-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" @@ -81,6 +81,7 @@ gemnasium-dependency_scanning: exists: !reference [.gemnasium-shared-rule, exists] variables: DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ exists: !reference [.gemnasium-shared-rule, exists] @@ -95,7 +96,7 @@ gemnasium-dependency_scanning: gemnasium-maven-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium-maven" rules: @@ -125,7 +126,7 @@ gemnasium-maven-dependency_scanning: gemnasium-python-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium-python" rules: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 9bb2ba69d84..c2d31fd9669 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -20,6 +20,11 @@ .review_ecs_base: stage: review extends: .deploy_to_ecs + after_script: + - echo "http://$(ecs get-task-hostname)" > environment_url.txt + artifacts: + paths: + - environment_url.txt .production_ecs_base: stage: production diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 4687a07d05b..34084272b29 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml - # This is an example .gitlab-ci.yml file to test (and optionally report the coverage # results of) your [Julia][1] packages. Please refer to the [documentation][2] # for more information about package development in Julia. @@ -12,6 +7,14 @@ # # [1]: http://julialang.org/ # [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml # Below is the template to run your tests in Julia .test_template: &test_definition diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 0ec67526234..3a490012f3d 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 2f214347ec3..65db649e22f 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml - # This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) # using the official mono docker image to build a visual studio project. # @@ -15,6 +10,14 @@ # # Please find the full example project here: # https://gitlab.com/tobiaskoch/gitlab-ci-example-mono +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml # see https://hub.docker.com/_/mono/ image: mono:latest diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 44370f896a7..7a4f7ed628b 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 7c8bbe464af..0eb3b483067 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 4edc003a638..12640d28d29 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 690a5a291e1..aab408aa830 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index 390f0bb8061..a83f84da818 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: @@ -26,3 +29,14 @@ test:cargo: script: - rustc --version && cargo --version # Print version info for debugging - cargo test --workspace --verbose + +# Optional: Use a third party library to generate gitlab junit reports +# test:junit-report: +# script: +# Should be specified in Cargo.toml +# - cargo install junitify +# - cargo test -- --format=json -Z unstable-options --report-time | junitify --out $CI_PROJECT_DIR/tests/ +# artifacts: +# when: always +# reports: +# junit: $CI_PROJECT_DIR/tests/*.xml diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index de54d64dc42..26efe7a8908 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index eedb3b7a310..3c4533d603e 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 841f17767eb..50ce181095e 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -23,6 +23,9 @@ # You need to have the network drive mapped as Local System user for gitlab-runner service to see it # The best way to persist the mapping is via a scheduled task # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. # place project specific paths in variables to make the rest of the script more generic variables: diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 0b75c298167..58e840da713 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml - # This is a very simple template that mainly relies on FastLane to build and distribute your app. # Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ # You will also need fastlane and signing configuration for this to work, along with a MacOS runner. @@ -15,6 +10,14 @@ # https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional # detail on what to keep in mind in this scenario. +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. + +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml + stages: - build - test diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index a452cb197ae..95dff83506d 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -26,7 +26,6 @@ module Gitlab variables.concat(secret_instance_variables) variables.concat(secret_group_variables(environment: environment)) variables.concat(secret_project_variables(environment: environment)) - variables.concat(job.trigger_request.user_variables) if job.trigger_request variables.concat(pipeline.variables) variables.concat(pipeline_schedule_variables) end @@ -52,7 +51,7 @@ module Gitlab # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 # Until then, we need to make both the old and the new KUBECONFIG contexts available collection.concat(deployment_variables(environment: environment, job: job)) - template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.token).execute + template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.try(:token)).execute kubeconfig_yaml = collection['KUBECONFIG']&.value template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 15ebd506055..59acfa80258 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -16,6 +16,14 @@ module Gitlab end def execute + Gitlab::Ci::YamlProcessor::FeatureFlags.with_actor(project) do + parse_config + end + end + + private + + def parse_config if @config_content.blank? return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) end @@ -35,7 +43,9 @@ module Gitlab Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end - private + def project + @opts[:project] + end def run_logical_validations! @stages = @ci_config.stages diff --git a/lib/gitlab/ci/yaml_processor/feature_flags.rb b/lib/gitlab/ci/yaml_processor/feature_flags.rb new file mode 100644 index 00000000000..f03db9d0e6b --- /dev/null +++ b/lib/gitlab/ci/yaml_processor/feature_flags.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class YamlProcessor + module FeatureFlags + ACTOR_KEY = 'ci_yaml_processor_feature_flag_actor' + NO_ACTOR_VALUE = :no_actor + + NoActorError = Class.new(StandardError) + NO_ACTOR_MESSAGE = "Actor not set. Ensure to call `enabled?` inside `with_actor` block" + + class << self + # Cache a feature flag actor as thread local variable so + # we can have it available later with #enabled? + def with_actor(actor) + previous = Thread.current[ACTOR_KEY] + + # When actor is `nil` the method `Thread.current[]=` does not + # create the ACTOR_KEY. Instead, we want to still save an explicit + # value to know that we are within the `with_actor` block. + Thread.current[ACTOR_KEY] = actor || NO_ACTOR_VALUE + + yield + ensure + Thread.current[ACTOR_KEY] = previous + end + + # Use this to check if a feature flag is enabled + def enabled?(feature_flag) + ::Feature.enabled?(feature_flag, current_actor) + end + + private + + def current_actor + value = Thread.current[ACTOR_KEY] || (raise NoActorError, NO_ACTOR_MESSAGE) + return if value == NO_ACTOR_VALUE + + value + rescue NoActorError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + + nil + end + end + end + end + end +end |