diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-19 22:11:55 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-11-19 22:11:55 +0000 |
| commit | 5a8431feceba47fd8e1804d9aa1b1730606b71d5 (patch) | |
| tree | e5df8e0ceee60f4af8093f5c4c2f934b8abced05 /lib/gitlab/ci | |
| parent | 4d477238500c347c6553d335d920bedfc5a46869 (diff) | |
| download | gitlab-ce-5a8431feceba47fd8e1804d9aa1b1730606b71d5.tar.gz | |
Add latest changes from gitlab-org/gitlab@12-5-stable-ee
Diffstat (limited to 'lib/gitlab/ci')
53 files changed, 1020 insertions, 295 deletions
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 8d25b66af9c..cbda3808b86 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -22,11 +22,11 @@ module Gitlab start_offset = @state.offset - @state.set_current_line!(style: Style.new(@state.inherited_style)) + @state.new_line!( + style: Style.new(@state.inherited_style)) stream.each_line do |line| - s = StringScanner.new(line) - convert_line(s) + consume_line(line) end # This must be assigned before flushing the current line @@ -52,26 +52,41 @@ module Gitlab private - def convert_line(scanner) - until scanner.eos? - - if scanner.scan(Gitlab::Regex.build_trace_section_regex) - handle_section(scanner) - elsif scanner.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(scanner) - elsif scanner.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif scanner.scan(/</) - @state.current_line << '<' - elsif scanner.scan(/\r?\n/) - # we advance the offset of the next current line - # so it does not start from \n - flush_current_line(advance_offset: scanner.matched_size) - else - @state.current_line << scanner.scan(/./m) - end - - @state.offset += scanner.matched_size + def consume_line(line) + scanner = StringScanner.new(line) + + consume_token(scanner) until scanner.eos? + end + + def consume_token(scanner) + if scan_token(scanner, Gitlab::Regex.build_trace_section_regex, consume: false) + handle_section(scanner) + elsif scan_token(scanner, /\e([@-_])(.*?)([@-~])/) + handle_sequence(scanner) + elsif scan_token(scanner, /\e(([@-_])(.*?)?)?$/) + # stop scanning + scanner.terminate + elsif scan_token(scanner, /\r?\n/) + flush_current_line + elsif scan_token(scanner, /\r/) + # drop last line + @state.current_line.clear! + elsif scan_token(scanner, /.[^\e\r\ns]*/m) + # this is a join from all previous tokens and first letters + # it always matches at least one character `.` + # it matches everything that is not start of: + # `\e`, `<`, `\r`, `\n`, `s` (for section_start) + @state.current_line << scanner[0] + else + raise 'invalid parser state' + end + end + + def scan_token(scanner, match, consume: true) + scanner.scan(match).tap do |result| + # we need to move offset as soon + # as we match the token + @state.offset += scanner.matched_size if consume && result end end @@ -96,32 +111,50 @@ module Gitlab section_name = sanitize_section_name(section) if action == "start" - handle_section_start(section_name, timestamp) + handle_section_start(scanner, section_name, timestamp) elsif action == "end" - handle_section_end(section_name, timestamp) + handle_section_end(scanner, section_name, timestamp) + else + raise 'unsupported action' end end - def handle_section_start(section, timestamp) - flush_current_line unless @state.current_line.empty? + def handle_section_start(scanner, section, timestamp) + # We make a new line for new section + flush_current_line + @state.open_section(section, timestamp) + + # we need to consume match after handling + # the open of section, as we want the section + # marker to be refresh on incremental update + @state.offset += scanner.matched_size end - def handle_section_end(section, timestamp) + def handle_section_end(scanner, section, timestamp) return unless @state.section_open?(section) - flush_current_line unless @state.current_line.empty? + # We flush the content to make the end + # of section to be a new line + flush_current_line + @state.close_section(section, timestamp) - # ensure that section end is detached from the last - # line in the section + # we need to consume match before handling + # as we want the section close marker + # not to be refreshed on incremental update + @state.offset += scanner.matched_size + + # this flushes an empty line with `section_duration` flush_current_line end - def flush_current_line(advance_offset: 0) - @lines << @state.current_line.to_h + def flush_current_line + unless @state.current_line.empty? + @lines << @state.current_line.to_h + end - @state.set_current_line!(advance_offset: advance_offset) + @state.new_line! end def sanitize_section_name(section) diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 173fb1df88e..21aa1f84353 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -47,12 +47,17 @@ module Gitlab @current_segment.text << data end + def clear! + @segments.clear + @current_segment = Segment.new(style: style) + end + def style @current_segment.style end def empty? - @segments.empty? && @current_segment.empty? + @segments.empty? && @current_segment.empty? && @section_duration.nil? end def update_style(ansi_commands) diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index db7a9035b8b..7e1a8102a35 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -46,9 +46,9 @@ module Gitlab @open_sections.key?(section) end - def set_current_line!(style: nil, advance_offset: 0) + def new_line!(style: nil) new_line = Line.new( - offset: @offset + advance_offset, + offset: @offset, style: style || @current_line.style, sections: @open_sections.keys ) diff --git a/lib/gitlab/ci/ansi2json/style.rb b/lib/gitlab/ci/ansi2json/style.rb index 2739ffdfa5d..77f61178b37 100644 --- a/lib/gitlab/ci/ansi2json/style.rb +++ b/lib/gitlab/ci/ansi2json/style.rb @@ -15,14 +15,10 @@ module Gitlab end def update(ansi_commands) - command = ansi_commands.shift - return unless command - - if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes - apply_changes(changes) - end + # treat e\[m as \e[0m + ansi_commands = ['0'] if ansi_commands.empty? - update(ansi_commands) + evaluate_stack_command(ansi_commands) end def set? @@ -50,6 +46,17 @@ module Gitlab private + def evaluate_stack_command(ansi_commands) + command = ansi_commands.shift + return unless command + + if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes + apply_changes(changes) + end + + evaluate_stack_command(ansi_commands) + end + def apply_changes(changes) case when changes[:reset] diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb new file mode 100644 index 00000000000..02b97ea76e9 --- /dev/null +++ b/lib/gitlab/ci/build/context/base.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Context + class Base + attr_reader :pipeline + + def initialize(pipeline) + @pipeline = pipeline + end + + def variables + raise NotImplementedError + end + + protected + + def pipeline_attributes + { + pipeline: pipeline, + project: pipeline.project, + user: pipeline.user, + ref: pipeline.ref, + tag: pipeline.tag, + trigger_request: pipeline.legacy_trigger, + protected: pipeline.protected_ref? + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb new file mode 100644 index 00000000000..dfd86d3ad72 --- /dev/null +++ b/lib/gitlab/ci/build/context/build.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Context + class Build < Base + include Gitlab::Utils::StrongMemoize + + attr_reader :attributes + + def initialize(pipeline, attributes = {}) + super(pipeline) + + @attributes = attributes + end + + def variables + strong_memoize(:variables) do + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate rules before we persist a Build + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. + stub_build.scoped_variables_hash + end + end + + private + + def stub_build + ::Ci::Build.new(build_attributes) + end + + def build_attributes + attributes.merge(pipeline_attributes) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/context/global.rb b/lib/gitlab/ci/build/context/global.rb new file mode 100644 index 00000000000..fdd3ac358d5 --- /dev/null +++ b/lib/gitlab/ci/build/context/global.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Context + class Global < Base + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline, yaml_variables:) + super(pipeline) + + @yaml_variables = yaml_variables.to_a + end + + def variables + strong_memoize(:variables) do + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate workflow:rules + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. + stub_build.scoped_variables_hash + .reject { |key, _value| key =~ /\ACI_(JOB|BUILD)/ } + end + end + + private + + def stub_build + ::Ci::Build.new(build_attributes) + end + + def build_attributes + pipeline_attributes.merge( + yaml_variables: @yaml_variables) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb index 9c705a1cd3e..9ae4198bbf7 100644 --- a/lib/gitlab/ci/build/policy/changes.rb +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -9,7 +9,7 @@ module Gitlab @globs = Array(globs) end - def satisfied_by?(pipeline, seed) + def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index 4c7dc947cd0..4e8693724e5 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -11,7 +11,7 @@ module Gitlab end end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index c3005303fd8..afe0ccb361e 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -9,7 +9,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) @patterns.any? do |pattern| pattern, path = pattern.split('@', 2) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index ceb5210cfb5..1394340ce1f 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -17,7 +17,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index e9c8864123f..7b1ce6330f0 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -9,8 +9,8 @@ module Gitlab @expressions = Array(expressions) end - def satisfied_by?(pipeline, seed) - variables = seed.scoped_variables_hash + def satisfied_by?(pipeline, context) + variables = context.variables statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index 43399c74457..c705b6f86c7 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -13,17 +13,21 @@ module Gitlab options: { start_in: start_in }.compact }.compact end + + def pass? + self.when != 'never' + end end - def initialize(rule_hashes, default_when = 'on_success') + def initialize(rule_hashes, default_when:) @rule_list = Rule.fabricate_list(rule_hashes) @default_when = default_when end - def evaluate(pipeline, build) + def evaluate(pipeline, context) if @rule_list.nil? Result.new(@default_when) - elsif matched_rule = match_rule(pipeline, build) + elsif matched_rule = match_rule(pipeline, context) Result.new( matched_rule.attributes[:when] || @default_when, matched_rule.attributes[:start_in] @@ -35,8 +39,8 @@ module Gitlab private - def match_rule(pipeline, build) - @rule_list.find { |rule| rule.matches?(pipeline, build) } + def match_rule(pipeline, context) + @rule_list.find { |rule| rule.matches?(pipeline, context) } end end end diff --git a/lib/gitlab/ci/build/rules/rule.rb b/lib/gitlab/ci/build/rules/rule.rb index 8d52158c8d2..077e4d150fb 100644 --- a/lib/gitlab/ci/build/rules/rule.rb +++ b/lib/gitlab/ci/build/rules/rule.rb @@ -23,8 +23,8 @@ module Gitlab end end - def matches?(pipeline, build) - @clauses.all? { |clause| clause.satisfied_by?(pipeline, build) } + def matches?(pipeline, context) + @clauses.all? { |clause| clause.satisfied_by?(pipeline, context) } end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb index bf787fe95a6..6d4bbbb8c21 100644 --- a/lib/gitlab/ci/build/rules/rule/clause.rb +++ b/lib/gitlab/ci/build/rules/rule/clause.rb @@ -20,7 +20,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline, seed = nil) + def satisfied_by?(pipeline, context = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 81d2ee6c24c..728a66ca87f 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -8,7 +8,7 @@ module Gitlab @globs = Array(globs) end - def satisfied_by?(pipeline, seed) + def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index 62f8371283f..85e77438f51 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -15,7 +15,7 @@ module Gitlab @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) end - def satisfied_by?(pipeline, seed) + def satisfied_by?(pipeline, context) paths = worktree_paths(pipeline) exact_matches?(paths) || pattern_matches?(paths) diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb index 18c3b450f95..6143a736ca6 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/if.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -8,10 +8,9 @@ module Gitlab @expression = expression end - def satisfied_by?(pipeline, seed) - variables = seed.scoped_variables_hash - - ::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful? + def satisfied_by?(pipeline, context) + ::Gitlab::Ci::Pipeline::Expression::Statement.new( + @expression, context.variables).truthful? end end end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 41613369ca2..9d8d7675234 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,7 +12,9 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths reports when expire_in].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as].freeze + EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze + EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" attributes ALLOWED_KEYS @@ -21,11 +23,18 @@ module Gitlab validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS + validates :paths, presence: true, if: :expose_as_present? with_options allow_nil: true do validates :name, type: String validates :untracked, boolean: true validates :paths, array_of_strings: true + validates :paths, array_of_strings: { + with: /\A[^*]*\z/, + message: "can't contain '*' when used with 'expose_as'" + }, if: :expose_as_present? + validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present? + validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? validates :reports, type: Hash validates :when, inclusion: { in: %w[on_success on_failure always], @@ -41,6 +50,12 @@ module Gitlab @config[:reports] = reports_value if @config.key?(:reports) @config end + + def expose_as_present? + return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true) + + !@config[:expose_as].nil? + end end end end diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb new file mode 100644 index 00000000000..10619ef9f8d --- /dev/null +++ b/lib/gitlab/ci/config/entry/boolean.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the interrutible value. + # + class Boolean < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, boolean: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 02e368c1813..7a86fca3056 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -11,11 +11,11 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable validations do - validates :config, array_of_strings_or_string: true + validates :config, string_or_nested_array_of_strings: true end def value - Array(@config) + Array(@config).flatten(1) end end end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index 6200d7c7f87..83127bde6e4 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -11,11 +11,10 @@ module Gitlab # class Default < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - - DuplicateError = Class.new(Gitlab::Config::Loader::FormatError) + include ::Gitlab::Config::Entry::Inheritable ALLOWED_KEYS = %i[before_script image services - after_script cache].freeze + after_script cache interruptible].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -41,31 +40,22 @@ module Gitlab description: 'Configure caching between build jobs.', inherit: true - helpers :before_script, :image, :services, :after_script, :cache - - def compose!(deps = nil) - super(self) + entry :interruptible, Entry::Boolean, + description: 'Set jobs interruptible default value.', + inherit: false - inherit!(deps) - end + helpers :before_script, :image, :services, :after_script, :cache, :interruptible private - def inherit!(deps) - return unless deps - - self.class.nodes.each do |key, factory| - next unless factory.inheritable? + def overwrite_entry(deps, key, current_entry) + inherited_entry = deps[key] - root_entry = deps[key] - next unless root_entry.specified? - - if self[key].specified? - raise DuplicateError, "#{key} is defined in top-level and `default:` entry" - end - - @entries[key] = root_entry + if inherited_entry.specified? && current_entry.specified? + raise InheritError, "#{key} is defined in top-level and `default:` entry" end + + inherited_entry unless current_entry.specified? end end end diff --git a/lib/gitlab/ci/config/entry/files.rb b/lib/gitlab/ci/config/entry/files.rb new file mode 100644 index 00000000000..d0d6a36d754 --- /dev/null +++ b/lib/gitlab/ci/config/entry/files.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an array of file paths. + # + class Files < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, array_of_strings: true + validates :config, length: { + minimum: 1, + maximum: 2, + too_short: 'requires at least %{count} item', + too_long: 'has too many items (maximum is %{count})' + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 07d5be86b1e..c75ae87a985 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -10,6 +10,7 @@ module Gitlab class Job < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Inheritable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script only except rules type image services @@ -37,7 +38,6 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true - validates :interruptible, boolean: true validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, less_than_or_equal_to: 50 } @@ -49,7 +49,6 @@ module Gitlab validates :timeout, duration: { limit: ChronicDuration.output(Project::MAX_BUILD_TIMEOUT) } validates :dependencies, array_of_strings: true - validates :needs, array_of_strings: true validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true end @@ -73,13 +72,16 @@ module Gitlab inherit: true entry :script, Entry::Commands, - description: 'Commands that will be executed in this job.' + description: 'Commands that will be executed in this job.', + inherit: false entry :stage, Entry::Stage, - description: 'Pipeline stage this job will be executed into.' + description: 'Pipeline stage this job will be executed into.', + inherit: false entry :type, Entry::Stage, - description: 'Deprecated: stage this job will be executed into.' + description: 'Deprecated: stage this job will be executed into.', + inherit: false entry :after_script, Entry::Script, description: 'Commands that will be executed when finishing job.', @@ -97,30 +99,50 @@ module Gitlab description: 'Services that will be used to execute this job.', inherit: true + entry :interruptible, Entry::Boolean, + description: 'Set jobs interruptible value.', + inherit: true + entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.', - default: Entry::Policy::DEFAULT_ONLY + default: Entry::Policy::DEFAULT_ONLY, + inherit: false entry :except, Entry::Policy, - description: 'Refs policy this job will be executed for.' + description: 'Refs policy this job will be executed for.', + inherit: false entry :rules, Entry::Rules, - description: 'List of evaluable Rules to determine job inclusion.' + description: 'List of evaluable Rules to determine job inclusion.', + inherit: false, + metadata: { + allowed_when: %w[on_success on_failure always never manual delayed].freeze + } + + entry :needs, Entry::Needs, + description: 'Needs configuration for this job.', + metadata: { allowed_needs: %i[job] }, + inherit: false entry :variables, Entry::Variables, - description: 'Environment variables available for this job.' + description: 'Environment variables available for this job.', + inherit: false entry :artifacts, Entry::Artifacts, - description: 'Artifacts configuration for this job.' + description: 'Artifacts configuration for this job.', + inherit: false entry :environment, Entry::Environment, - description: 'Environment configuration for this job.' + description: 'Environment configuration for this job.', + inherit: false entry :coverage, Entry::Coverage, - description: 'Coverage configuration for this job.' + description: 'Coverage configuration for this job.', + inherit: false entry :retry, Entry::Retry, - description: 'Retry configuration for this job.' + description: 'Retry configuration for this job.', + inherit: false helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, @@ -155,8 +177,6 @@ module Gitlab @entries.delete(:except) end end - - inherit!(deps) end def name @@ -185,21 +205,8 @@ module Gitlab private - # We inherit config entries from `default:` - # if the entry has the `inherit: true` flag set - def inherit!(deps) - return unless deps - - self.class.nodes.each do |key, factory| - next unless factory.inheritable? - - default_entry = deps.default[key] - job_entry = self[key] - - if default_entry.specified? && !job_entry.specified? - @entries[key] = default_entry - end - end + def overwrite_entry(deps, key, current_entry) + deps.default[key] unless current_entry.specified? end def to_hash diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index 0c10967e629..f12f0919348 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -7,11 +7,48 @@ module Gitlab ## # Entry that represents a key. # - class Key < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable + class Key < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleKey, if: -> (config) { config.is_a?(String) || config.is_a?(Symbol) } + strategy :ComplexKey, if: -> (config) { config.is_a?(Hash) } - validations do - validates :config, key: true + class SimpleKey < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, key: true + end + + def self.default + 'default' + end + + def value + super.to_s + end + end + + class ComplexKey < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[files prefix].freeze + REQUIRED_KEYS = %i[files].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_KEYS + end + + entry :files, Entry::Files, + description: 'Files that should be used to build the key' + entry :prefix, Entry::Prefix, + description: 'Prefix that is added to the final cache key' + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be a hash, a string or a symbol"] + end end def self.default diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb new file mode 100644 index 00000000000..b6db546d8ff --- /dev/null +++ b/lib/gitlab/ci/config/entry/need.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Need < ::Gitlab::Config::Entry::Simplifiable + strategy :Job, if: -> (config) { config.is_a?(String) } + + class Job < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: String + end + + def type + :job + end + + def value + { name: @config } + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def type + end + + def value + end + + def errors + ["#{location} has an unsupported type"] + end + end + end + end + end + end +end + +::Gitlab::Ci::Config::Entry::Need.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Need') diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb new file mode 100644 index 00000000000..28452aaaa16 --- /dev/null +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a set of needs dependencies. + # + class Needs < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + end + + validate on: :composed do + extra_keys = value.keys - opt(:allowed_needs) + if extra_keys.any? + errors.add(:config, "uses invalid types: #{extra_keys.join(', ')}") + end + end + end + + def compose!(deps = nil) + super(deps) do + [@config].flatten.each_with_index do |need, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Need) + .value(need) + .with(key: "need", parent: self, description: "need definition.") # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + + def value + values = @entries.values.select(&:type) + values.group_by(&:type).transform_values do |values| + values.map(&:value) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/prefix.rb b/lib/gitlab/ci/config/entry/prefix.rb new file mode 100644 index 00000000000..3244ad6d611 --- /dev/null +++ b/lib/gitlab/ci/config/entry/prefix.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a key prefix. + # + class Prefix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, key: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 07022ff7b54..25fb278d9b8 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable ALLOWED_KEYS = %i[default include before_script image services - after_script variables stages types cache].freeze + after_script variables stages types cache workflow].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -64,6 +64,9 @@ module Gitlab description: 'Configure caching between build jobs.', reserved: true + entry :workflow, Entry::Workflow, + description: 'List of evaluable rules to determine Pipeline status' + helpers :default, :jobs, :stages, :types, :variables delegate :before_script_value, diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 5d6d1c026e3..59e0ef583ae 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -8,9 +8,9 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - CLAUSES = %i[if changes exists].freeze - ALLOWED_KEYS = %i[if changes exists when start_in].freeze - ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze + CLAUSES = %i[if changes exists].freeze + ALLOWED_KEYS = %i[if changes exists when start_in].freeze + ALLOWABLE_WHEN = %w[on_success on_failure always never manual delayed].freeze attributes :if, :changes, :exists, :when, :start_in @@ -25,7 +25,14 @@ module Gitlab with_options allow_nil: true do validates :if, expression: true validates :changes, :exists, array_of_strings: true, length: { maximum: 50 } - validates :when, allowed_values: { in: ALLOWED_WHEN } + validates :when, allowed_values: { in: ALLOWABLE_WHEN } + end + + validate do + validates_with Gitlab::Config::Entry::Validators::AllowedValuesValidator, + attributes: %i[when], + allow_nil: true, + in: opt(:allowed_when) end end diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb index 9d25a82b521..285e18218b3 100644 --- a/lib/gitlab/ci/config/entry/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -11,7 +11,11 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable validations do - validates :config, array_of_strings: true + validates :config, nested_array_of_strings: true + end + + def value + config.flatten(1) end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb new file mode 100644 index 00000000000..a51a3fbdcd2 --- /dev/null +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Workflow < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[rules].freeze + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, presence: true + end + + entry :rules, Entry::Rules, + description: 'List of evaluable Rules to determine Pipeline status.', + metadata: { allowed_when: %w[always never] } + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 09f9bf5f69f..e714ef225f5 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -18,8 +18,8 @@ module Gitlab config[:dependencies] = expand_names(config[:dependencies]) end - if config[:needs] - config[:needs] = expand_names(config[:needs]) + if job_needs = config.dig(:needs, :job) + config[:needs][:job] = expand_needs(job_needs) end config @@ -36,6 +36,22 @@ module Gitlab end end + def expand_needs(job_needs) + return unless job_needs + + job_needs.flat_map do |job_need| + job_need_name = job_need[:name].to_sym + + if all_job_names = parallelized_jobs[job_need_name] + all_job_names.map do |job_name| + { name: job_name } + end + else + job_need + end + end + end + def parallelized_jobs strong_memoize(:parallelized_jobs) do @jobs_config.each_with_object({}) do |(job_name, config), hash| diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index bab1c73e2f1..aabdf7ce47d 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Chain class Base - attr_reader :pipeline, :command + attr_reader :pipeline, :command, :config delegate :project, :current_user, to: :command diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 899df81ea5c..9662209f88e 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -22,8 +22,6 @@ module Gitlab external_pull_request: @command.external_pull_request, variables_attributes: Array(@command.variables_attributes) ) - - @pipeline.set_config_source end def break? diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 58f89a6be5e..c2df419cca0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,9 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update + :chat_data, :allow_mirror_update, + # These attributes are set by Chains during processing: + :config_content, :config_processor, :stage_seeds ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb new file mode 100644 index 00000000000..a8cd99b8e92 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content < Chain::Base + include Chain::Helpers + + def perform! + return if @command.config_content + + if content = content_from_repo + @command.config_content = content + @pipeline.config_source = :repository_source + # TODO: we should persist ci_config_path + # @pipeline.config_path = ci_config_path + elsif content = content_from_auto_devops + @command.config_content = content + @pipeline.config_source = :auto_devops_source + end + + unless @command.config_content + return error("Missing #{ci_config_path} file") + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + + private + + def content_from_repo + return unless project + return unless @pipeline.sha + return unless ci_config_path + + project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path) + rescue GRPC::NotFound, GRPC::Internal + nil + end + + def content_from_auto_devops + return unless project&.auto_devops_enabled? + + Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content + end + + def ci_config_path + project.ci_config_path.presence || '.gitlab-ci.yml' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb new file mode 100644 index 00000000000..731b0fdb286 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Process < Chain::Base + include Chain::Helpers + + def perform! + raise ArgumentError, 'missing config content' unless @command.config_content + + @command.config_processor = ::Gitlab::Ci::YamlProcessor.new( + @command.config_content, { + project: project, + sha: @pipeline.sha, + user: current_user + } + ) + rescue Gitlab::Ci::YamlProcessor::ValidationError => ex + error(ex.message, config_error: true) + rescue => ex + Gitlab::Sentry.track_acceptable_exception(ex, extra: { + project_id: project.id, + sha: @pipeline.sha + }) + + error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})", + config_error: true) + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb new file mode 100644 index 00000000000..0ee9485eebc --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EvaluateWorkflowRules < Chain::Base + include ::Gitlab::Utils::StrongMemoize + include Chain::Helpers + + def perform! + return unless Feature.enabled?(:workflow_rules, @pipeline.project) + + unless workflow_passed? + error('Pipeline filtered out by workflow rules.') + end + end + + def break? + return false unless Feature.enabled?(:workflow_rules, @pipeline.project) + + !workflow_passed? + end + + private + + def workflow_passed? + strong_memoize(:workflow_passed) do + workflow_rules.evaluate(@pipeline, global_context).pass? + end + end + + def workflow_rules + Gitlab::Ci::Build::Rules.new( + workflow_config[:rules], default_when: 'always') + end + + def global_context + Gitlab::Ci::Build::Context::Global.new( + @pipeline, yaml_variables: workflow_config[:yaml_variables]) + end + + def workflow_config + @command.config_processor.workflow_attributes || {} + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 13eca5a9d28..3a40c7b167c 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -10,29 +10,12 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! - # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - - # Protect the pipeline. This is assigned in Populate instead of - # Build to prevent erroring out on ambiguous refs. - pipeline.protected = @command.protected_ref? - - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) - - ## - # Gather all runtime build/stage errors - # - if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence - return error(seeds_errors.join("\n"), config_error: true) - end + raise ArgumentError, 'missing stage seeds' unless @command.stage_seeds ## # Populate pipeline with all stages, and stages with builds. # - pipeline.stages = pipeline.stage_seeds.map(&:to_resource) + pipeline.stages = @command.stage_seeds.map(&:to_resource) if pipeline.stages.none? return error('No stages / jobs for this pipeline.') diff --git a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb index 1e09b417311..9267c72efa4 100644 --- a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb +++ b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb @@ -6,11 +6,13 @@ module Gitlab module Chain class RemoveUnwantedChatJobs < Chain::Base def perform! - return unless pipeline.config_processor && pipeline.chat? + raise ArgumentError, 'missing config processor' unless @command.config_processor + + return unless pipeline.chat? # When scheduling a chat pipeline we only want to run the build # that matches the chat command. - pipeline.config_processor.jobs.select! do |name, _| + @command.config_processor.jobs.select! do |name, _| name.to_s == command.chat_data[:command].to_s end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb new file mode 100644 index 00000000000..2e177cfec7e --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Seed < Chain::Base + include Chain::Helpers + include Gitlab::Utils::StrongMemoize + + def perform! + raise ArgumentError, 'missing config processor' unless @command.config_processor + + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + + # Protect the pipeline. This is assigned in Populate instead of + # Build to prevent erroring out on ambiguous refs. + pipeline.protected = @command.protected_ref? + + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + ## + # Gather all runtime build/stage errors + # + if stage_seeds_errors + return error(stage_seeds_errors.join("\n"), config_error: true) + end + + @command.stage_seeds = stage_seeds + end + + def break? + pipeline.errors.any? + end + + private + + def stage_seeds_errors + stage_seeds.flat_map(&:errors).compact.presence + end + + def stage_seeds + strong_memoize(:stage_seeds) do + seeds = stages_attributes.inject([]) do |previous_stages, attributes| + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(pipeline, attributes, previous_stages) + previous_stages + [seed] + end + + seeds.select(&:included?) + end + end + + def stages_attributes + @command.config_processor.stages_attributes + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb deleted file mode 100644 index 28c38cc3d18..00000000000 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Validate - class Config < Chain::Base - include Chain::Helpers - - def perform! - unless @pipeline.config_processor - unless @pipeline.ci_yaml_file - return error("Missing #{@pipeline.ci_yaml_file_path} file") - end - - if @command.save_incompleted && @pipeline.has_yaml_errors? - @pipeline.drop!(:config_error) - end - - error(@pipeline.yaml_errors) - end - end - - def break? - @pipeline.errors.any? || @pipeline.persisted? - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index fc9c540088b..dce56b22666 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -28,7 +28,9 @@ module Gitlab @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules - .new(attributes.delete(:rules)) + .new(attributes.delete(:rules), default_when: 'on_success') + @cache = Seed::Build::Cache + .new(pipeline, attributes.delete(:cache)) end def name @@ -38,7 +40,7 @@ module Gitlab def included? strong_memoize(:inclusion) do if @using_rules - included_by_rules? + rules_result.pass? elsif @using_only || @using_except all_of_only? && none_of_except? else @@ -59,6 +61,7 @@ module Gitlab @seed_attributes .deep_merge(pipeline_attributes) .deep_merge(rules_attributes) + .deep_merge(cache_attributes) end def bridge? @@ -80,26 +83,14 @@ module Gitlab end end - def scoped_variables_hash - strong_memoize(:scoped_variables_hash) do - # This is a temporary piece of technical debt to allow us access - # to the CI variables to evaluate rules before we persist a Build - # with the result. We should refactor away the extra Build.new, - # but be able to get CI Variables directly from the Seed::Build. - ::Ci::Build.new( - @seed_attributes.merge(pipeline_attributes) - ).scoped_variables_hash - end - end - private def all_of_only? - @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } + @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end def none_of_except? - @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + @except.none? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end def needs_errors @@ -141,13 +132,27 @@ module Gitlab } end - def included_by_rules? - rules_attributes[:when] != 'never' + def rules_attributes + return {} unless @using_rules + + rules_result.build_attributes end - def rules_attributes - strong_memoize(:rules_attributes) do - @using_rules ? @rules.evaluate(@pipeline, self).build_attributes : {} + def rules_result + strong_memoize(:rules_result) do + @rules.evaluate(@pipeline, evaluate_context) + end + end + + def evaluate_context + strong_memoize(:evaluate_context) do + Gitlab::Ci::Build::Context::Build.new(@pipeline, @seed_attributes) + end + end + + def cache_attributes + strong_memoize(:cache_attributes) do + @cache.build_attributes end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb new file mode 100644 index 00000000000..7671035b896 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Build + class Cache + def initialize(pipeline, cache) + @pipeline = pipeline + local_cache = cache.to_h.deep_dup + @key = local_cache.delete(:key) + @paths = local_cache.delete(:paths) + @policy = local_cache.delete(:policy) + @untracked = local_cache.delete(:untracked) + + raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? + end + + def build_attributes + { + options: { + cache: { + key: key_string, + paths: @paths, + policy: @policy, + untracked: @untracked + }.compact.presence + }.compact + } + end + + private + + def key_string + key_from_string || key_from_files + end + + def key_from_string + @key.to_s if @key.is_a?(String) || @key.is_a?(Symbol) + end + + def key_from_files + return unless @key.is_a?(Hash) + + [@key[:prefix], files_digest].select(&:present?).join('-') + end + + def files_digest + hash_of_the_latest_changes || 'default' + end + + def hash_of_the_latest_changes + return unless Feature.enabled?(:ci_file_based_cache, @pipeline.project, default_enabled: true) + + ids = files.map { |path| last_commit_id_for_path(path) } + ids = ids.compact.sort.uniq + + Digest::SHA1.hexdigest(ids.join('-')) if ids.any? + end + + def files + @key[:files] + .to_a + .select(&:present?) + .uniq + end + + def last_commit_id_for_path(path) + @pipeline.project.repository.last_commit_id_for_path(@pipeline.sha, path) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 961012c2cee..910d93f54ce 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -16,7 +16,9 @@ module Gitlab stale_schedule: 'stale schedule', job_execution_timeout: 'job execution timeout', archived_failure: 'archived failure', - unmet_prerequisites: 'unmet prerequisites' + unmet_prerequisites: 'unmet prerequisites', + scheduler_failure: 'scheduler failure', + data_integrity_failure: 'data integrity failure' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 3cdb7b5420c..a60b00b2ee8 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -18,7 +18,7 @@ code_quality: --env SOURCE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/security-products/codequality:12-0-stable" /code + "registry.gitlab.com/gitlab-org/security-products/codequality:12-5-stable" /code artifacts: reports: codequality: gl-code-quality-report.json diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index ae2ff9992f9..7a672f910dd 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,8 +1,8 @@ -.auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" +.dast-auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.6.0" dast_environment_deploy: - extends: .auto-deploy + extends: .dast-auto-deploy stage: review script: - auto-deploy check_kube_domain @@ -28,10 +28,10 @@ dast_environment_deploy: variables: - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - - $DAST_WEBSITE # we don't need to create a review app if a URL is already given + - $DAST_WEBSITE # we don't need to create a review app if a URL is already given stop_dast_environment: - extends: .auto-deploy + extends: .dast-auto-deploy stage: cleanup variables: GIT_STRATEGY: none diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index a8ec2d4781d..738be44d5f4 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.7.0" review: extends: .auto-deploy diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index f058468ed8e..ef2fc561201 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -9,16 +9,17 @@ container_scanning: name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION entrypoint: [] variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here - # with a specific version to provide consistency for integration testing purposes - CLAIR_DB_IMAGE_TAG: latest - # Override this variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yaml` file. - # See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image + # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes + CLAIR_DB_IMAGE_TAG: "latest" + CLAIR_DB_IMAGE: "arminc/clair-db:$CLAIR_DB_IMAGE_TAG" + # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` + # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none allow_failure: true services: - - name: arminc/clair-db:$CLAIR_DB_IMAGE_TAG + - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: # the kubernetes executor currently ignores the Docker image entrypoint value, so the start.sh script must diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index c8930bc6263..4993d22d400 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -4,6 +4,12 @@ # List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +variables: + DS_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + DS_DEFAULT_ANALYZERS: "gemnasium, retire.js, gemnasium-python, gemnasium-maven, bundler-audit" + DS_MAJOR_VERSION: 2 + DS_DISABLE_DIND: "false" + dependency_scanning: stage: test image: docker:stable @@ -45,6 +51,7 @@ dependency_scanning: DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ + MAVEN_CLI_OPTS \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ @@ -61,3 +68,63 @@ dependency_scanning: except: variables: - $DEPENDENCY_SCANNING_DISABLED + - $DS_DISABLE_DIND == 'true' + +.analyzer: + extends: dependency_scanning + services: [] + except: + variables: + - $DS_DISABLE_DIND == 'false' + script: + - /analyzer run + +gemnasium-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php/ + +gemnasium-maven-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjava\b/ + +gemnasium-python-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ + +bundler-audit-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/bundler-audit:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ + +retire-js-dependency_scanning: + extends: .analyzer + image: + name: "$DS_ANALYZER_IMAGE_PREFIX/retire.js:$DS_MAJOR_VERSION" + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $DS_DEFAULT_ANALYZERS =~ /retire.js/ && + $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a0c2ab3aa26..c81b4efddbc 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex" - SAST_MAJOR_VERSION: 2 + SAST_ANALYZER_IMAGE_TAG: 2 SAST_DISABLE_DIND: "false" sast: @@ -35,45 +35,12 @@ sast: export DOCKER_HOST='tcp://localhost:2375' fi fi - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } + - | + printenv | grep -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | cut -d'=' -f1 | \ + (while IFS='\\n' read -r VAR; do unset -v "$VAR"; done; /bin/printenv > .env) - | docker run \ - $(propagate_env_vars \ - SAST_BANDIT_EXCLUDED_PATHS \ - SAST_ANALYZER_IMAGES \ - SAST_ANALYZER_IMAGE_PREFIX \ - SAST_ANALYZER_IMAGE_TAG \ - SAST_DEFAULT_ANALYZERS \ - SAST_PULL_ANALYZER_IMAGES \ - SAST_BRAKEMAN_LEVEL \ - SAST_FLAWFINDER_LEVEL \ - SAST_GITLEAKS_ENTROPY_LEVEL \ - SAST_GOSEC_LEVEL \ - SAST_EXCLUDED_PATHS \ - SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ - SAST_RUN_ANALYZER_TIMEOUT \ - SAST_JAVA_VERSION \ - ANT_HOME \ - ANT_PATH \ - GRADLE_PATH \ - JAVA_OPTS \ - JAVA_PATH \ - JAVA_8_VERSION \ - JAVA_11_VERSION \ - MAVEN_CLI_OPTS \ - MAVEN_PATH \ - MAVEN_REPO_PATH \ - SBT_PATH \ - FAIL_NEVER \ - ) \ + --env-file .env \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code @@ -94,7 +61,7 @@ sast: bandit-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -104,7 +71,7 @@ bandit-sast: brakeman-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -114,7 +81,7 @@ brakeman-sast: eslint-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -124,7 +91,7 @@ eslint-sast: flawfinder-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -134,7 +101,7 @@ flawfinder-sast: gosec-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -144,7 +111,7 @@ gosec-sast: nodejs-scan-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -154,7 +121,7 @@ nodejs-scan-sast: phpcs-security-audit-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -164,7 +131,7 @@ phpcs-security-audit-sast: pmd-apex-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -174,7 +141,7 @@ pmd-apex-sast: secrets-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -183,7 +150,7 @@ secrets-sast: security-code-scan-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -193,7 +160,7 @@ security-code-scan-sast: sobelow-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -203,7 +170,7 @@ sobelow-sast: spotbugs-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && @@ -213,7 +180,7 @@ spotbugs-sast: tslint-sast: extends: .analyzer image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_MAJOR_VERSION" + name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_ANALYZER_IMAGE_TAG" only: variables: - $GITLAB_FEATURES =~ /\bsast\b/ && diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index f6a3abefcfb..833c545fc5b 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -39,15 +39,15 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: yaml_variables(name), - needs_attributes: job[:needs]&.map { |need| { name: need } }, + yaml_variables: transform_to_yaml_variables(job_variables(name)), + needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], rules: job[:rules], + cache: job[:cache], options: { image: job[:image], services: job[:services], artifacts: job[:artifacts], - cache: job[:cache], dependencies: job[:dependencies], job_timeout: job[:timeout], before_script: job[:before_script], @@ -59,7 +59,7 @@ module Gitlab instance: job[:instance], start_in: job[:start_in], trigger: job[:trigger], - bridge_needs: job[:needs] + bridge_needs: job.dig(:needs, :bridge)&.first }.compact }.compact end @@ -83,6 +83,13 @@ module Gitlab end end + def workflow_attributes + { + rules: @config.dig(:workflow, :rules), + yaml_variables: transform_to_yaml_variables(@variables) + } + end + def self.validation_message(content, opts = {}) return 'Please provide content of .gitlab-ci.yml' if content.blank? @@ -118,20 +125,17 @@ module Gitlab end end - def yaml_variables(name) - variables = (@variables || {}) - .merge(job_variables(name)) + def job_variables(name) + job_variables = @jobs.dig(name.to_sym, :variables) - variables.map do |key, value| - { key: key.to_s, value: value, public: true } - end + @variables.to_h + .merge(job_variables.to_h) end - def job_variables(name) - job = @jobs[name.to_sym] - return {} unless job - - job[:variables] || {} + def transform_to_yaml_variables(variables) + variables.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end end def validate_job_stage!(name, job) @@ -159,17 +163,19 @@ module Gitlab end def validate_job_needs!(name, job) - return unless job[:needs] + return unless job.dig(:needs, :job) stage_index = @stages.index(job[:stage]) - job[:needs].each do |need| - raise ValidationError, "#{name} job: undefined need: #{need}" unless @jobs[need.to_sym] + job.dig(:needs, :job).each do |need| + need_job_name = need[:name] + + raise ValidationError, "#{name} job: undefined need: #{need_job_name}" unless @jobs[need_job_name.to_sym] - needs_stage_index = @stages.index(@jobs[need.to_sym][:stage]) + needs_stage_index = @stages.index(@jobs[need_job_name.to_sym][:stage]) unless needs_stage_index.present? && needs_stage_index < stage_index - raise ValidationError, "#{name} job: need #{need} is not defined in prior stages" + raise ValidationError, "#{name} job: need #{need_job_name} is not defined in prior stages" end end end |