diff options
Diffstat (limited to 'lib/gitlab/ci')
37 files changed, 1328 insertions, 157 deletions
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb new file mode 100644 index 00000000000..72b75791bbb --- /dev/null +++ b/lib/gitlab/ci/ansi2html.rb @@ -0,0 +1,344 @@ +# ANSI color library +# +# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code +module Gitlab + module Ci + module Ansi2html + # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) + COLOR = { + 0 => 'black', # not that this is gray in the intense color table + 1 => 'red', + 2 => 'green', + 3 => 'yellow', + 4 => 'blue', + 5 => 'magenta', + 6 => 'cyan', + 7 => 'white', # not that this is gray in the dark (aka default) color table + }.freeze + + STYLE_SWITCHES = { + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 + }.freeze + + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) + end + + class Converter + def on_0(s) reset() end + + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + + def on_9(s) enable(STYLE_SWITCHES[:cross]) end + + def on_21(s) disable(STYLE_SWITCHES[:bold]) end + + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + + def on_29(s) disable(STYLE_SWITCHES[:cross]) end + + def on_30(s) set_fg_color(0) end + + def on_31(s) set_fg_color(1) end + + def on_32(s) set_fg_color(2) end + + def on_33(s) set_fg_color(3) end + + def on_34(s) set_fg_color(4) end + + def on_35(s) set_fg_color(5) end + + def on_36(s) set_fg_color(6) end + + def on_37(s) set_fg_color(7) end + + def on_38(s) set_fg_color_256(s) end + + def on_39(s) set_fg_color(9) end + + def on_40(s) set_bg_color(0) end + + def on_41(s) set_bg_color(1) end + + def on_42(s) set_bg_color(2) end + + def on_43(s) set_bg_color(3) end + + def on_44(s) set_bg_color(4) end + + def on_45(s) set_bg_color(5) end + + def on_46(s) set_bg_color(6) end + + def on_47(s) set_bg_color(7) end + + def on_48(s) set_bg_color_256(s) end + + def on_49(s) set_bg_color(9) end + + def on_90(s) set_fg_color(0, 'l') end + + def on_91(s) set_fg_color(1, 'l') end + + def on_92(s) set_fg_color(2, 'l') end + + def on_93(s) set_fg_color(3, 'l') end + + def on_94(s) set_fg_color(4, 'l') end + + def on_95(s) set_fg_color(5, 'l') end + + def on_96(s) set_fg_color(6, 'l') end + + def on_97(s) set_fg_color(7, 'l') end + + def on_99(s) set_fg_color(9, 'l') end + + def on_100(s) set_bg_color(0, 'l') end + + def on_101(s) set_bg_color(1, 'l') end + + def on_102(s) set_bg_color(2, 'l') end + + def on_103(s) set_bg_color(3, 'l') end + + def on_104(s) set_bg_color(4, 'l') end + + def on_105(s) set_bg_color(5, 'l') end + + def on_106(s) set_bg_color(6, 'l') end + + def on_107(s) set_bg_color(7, 'l') end + + def on_109(s) set_bg_color(9, 'l') end + + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze + + def convert(stream, new_state) + reset_state + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset + + open_new_tag + + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(Gitlab::Regex.build_trace_section_regex) + handle_section(s) + elsif s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\r?\n/) + @out << '<br>' + else + @out << s.scan(/./m) + end + @offset += s.matched_size + end + end + + close_open_tags() + + OpenStruct.new( + html: @out.force_encoding(Encoding.default_external), + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) + end + + def handle_section(s) + action = s[1] + timestamp = s[2] + section = s[3] + line = s.matched()[0...-5] # strips \r\033[0K + + @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} + end + + def handle_sequence(s) + indicator = s[1] + commands = s[2].split ';' + terminator = s[3] + + # We are only interested in color and text style changes - triggered by + # sequences starting with '\e[' and ending with 'm'. Any other control + # sequence gets stripped (including stuff like "delete last line") + return unless indicator == '[' && terminator == 'm' + + close_open_tags() + + if commands.empty?() + reset() + return + end + + evaluate_command_stack(commands) + + open_new_tag + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend + end + + evaluate_command_stack(stack) + end + + def open_new_tag + css_classes = [] + + unless @fg_color.nil? + fg_color = @fg_color + # Most terminals show bold colored text in the light color variant + # Let's mimic that here + if @style_mask & STYLE_SWITCHES[:bold] != 0 + fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + end + css_classes << fg_color + end + css_classes << @bg_color unless @bg_color.nil? + + STYLE_SWITCHES.each do |css_class, flag| + css_classes << "term-#{css_class}" if @style_mask & flag != 0 + end + + return if css_classes.empty? + + @out << %{<span class="#{css_classes.join(' ')}">} + @n_open_tags += 1 + end + + def close_open_tags + while @n_open_tags > 0 + @out << %{</span>} + @n_open_tags -= 1 + end + end + + def reset_state + @offset = 0 + @n_open_tags = 0 + @out = '' + reset + end + + def state + state = STATE_PARAMS.inject({}) do |h, param| + h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend + h + end + Base64.urlsafe_encode64(state.to_json) + end + + def restore_state(new_state, stream) + state = Base64.urlsafe_decode64(new_state) + state = JSON.parse(state, symbolize_names: true) + return if state[:offset].to_i > stream.size + + STATE_PARAMS.each do |param| + send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def reset + @fg_color = nil + @bg_color = nil + @style_mask = 0 + end + + def enable(flag) + @style_mask |= flag + end + + def disable(flag) + @style_mask &= ~flag + end + + def set_fg_color(color_index, prefix = nil) + @fg_color = get_term_color_class(color_index, ["fg", prefix]) + end + + def set_bg_color(color_index, prefix = nil) + @bg_color = get_term_color_class(color_index, ["bg", prefix]) + end + + def get_term_color_class(color_index, prefix) + color_name = COLOR[color_index] + return nil if color_name.nil? + + get_color_class(["term", prefix, color_name]) + end + + def set_fg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "fg") + @fg_color = css_class unless css_class.nil? + end + + def set_bg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "bg") + @bg_color = css_class unless css_class.nil? + end + + def get_xterm_color_class(command_stack, prefix) + # the 38 and 48 commands have to be followed by "5" and the color index + return unless command_stack.length >= 2 + return unless command_stack[0] == "5" + + command_stack.shift() # ignore the "5" command + color_index = command_stack.shift().to_i + + return unless color_index >= 0 + return unless color_index <= 255 + + get_color_class(["xterm", prefix, color_index]) + end + + def get_color_class(segments) + [segments].flatten.compact.join('-') + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb new file mode 100644 index 00000000000..d10cc7802d4 --- /dev/null +++ b/lib/gitlab/ci/build/policy.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Build + module Policy + def self.fabricate(specs) + specifications = specs.to_h.map do |spec, value| + self.const_get(spec.to_s.camelize).new(value) + end + + specifications.compact + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb new file mode 100644 index 00000000000..b20d374288f --- /dev/null +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Build + module Policy + class Kubernetes < Policy::Specification + def initialize(spec) + unless spec.to_sym == :active + raise UnknownPolicyError + end + end + + def satisfied_by?(pipeline) + pipeline.has_kubernetes_active? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb new file mode 100644 index 00000000000..eadc0948d2f --- /dev/null +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -0,0 +1,43 @@ +module Gitlab + module Ci + module Build + module Policy + class Refs < Policy::Specification + def initialize(refs) + @patterns = Array(refs) + end + + def satisfied_by?(pipeline) + @patterns.any? do |pattern| + pattern, path = pattern.split('@', 2) + + matches_path?(path, pipeline) && + matches_pattern?(pattern, pipeline) + end + end + + private + + def matches_path?(path, pipeline) + return true unless path + + pipeline.project_full_path == path + end + + def matches_pattern?(pattern, pipeline) + return true if pipeline.tag? && pattern == 'tags' + return true if pipeline.branch? && pattern == 'branches' + return true if pipeline.source == pattern + return true if pipeline.source&.pluralize == pattern + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ pipeline.ref + else + pattern == pipeline.ref + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb new file mode 100644 index 00000000000..c317291f29d --- /dev/null +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Build + module Policy + ## + # Abstract class that defines an interface of job policy + # specification. + # + # Used for job's only/except policy configuration. + # + class Specification + UnknownPolicyError = Class.new(StandardError) + + def initialize(spec) + @spec = spec + end + + def satisfied_by?(pipeline) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb new file mode 100644 index 00000000000..7df7b542d91 --- /dev/null +++ b/lib/gitlab/ci/charts.rb @@ -0,0 +1,118 @@ +module Gitlab + module Ci + module Charts + module DailyInterval + def grouped_count(query) + query + .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") + .count(:created_at) + .transform_keys { |date| date.strftime(@format) } + end + + def interval_step + @interval_step ||= 1.day + end + end + + module MonthlyInterval + def grouped_count(query) + if Gitlab::Database.postgresql? + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) + else + query + .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") + .count(:created_at) + end + end + + def interval_step + @interval_step ||= 1.month + end + end + + class Chart + attr_reader :labels, :total, :success, :project, :pipeline_times + + def initialize(project) + @labels = [] + @total = [] + @success = [] + @pipeline_times = [] + @project = project + + collect + end + + def collect + query = project.pipelines + .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection + + totals_count = grouped_count(query) + success_count = grouped_count(query.success) + + current = @from + while current < @to + label = current.strftime(@format) + + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) + + current += interval_step + end + end + end + + class YearChart < Chart + include MonthlyInterval + + def initialize(*) + @to = Date.today.end_of_month + @from = @to.years_ago(1).beginning_of_month + @format = '%d %B %Y' + + super + end + end + + class MonthChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 30.days + @format = '%d %B' + + super + end + end + + class WeekChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 7.days + @format = '%d %B' + + super + end + end + + class PipelineTime < Chart + def collect + commits = project.pipelines.last(30) + + commits.each do |commit| + @labels << commit.short_sha + duration = commit.duration || 0 + @pipeline_times << (duration / 60) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb new file mode 100644 index 00000000000..0daddaa638c --- /dev/null +++ b/lib/gitlab/ci/mask_secret.rb @@ -0,0 +1,12 @@ +module Gitlab + module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end + end +end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb new file mode 100644 index 00000000000..3994a50772b --- /dev/null +++ b/lib/gitlab/ci/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Model + def table_name_prefix + "ci_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb new file mode 100644 index 00000000000..8d82e1b288d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Base + attr_reader :pipeline, :project, :current_user + + def initialize(pipeline, command) + @pipeline = pipeline + @command = command + + @project = command.project + @current_user = command.current_user + end + + def perform! + raise NotImplementedError + end + + def break? + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb new file mode 100644 index 00000000000..d5e17a123df --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Create < Chain::Base + include Chain::Helpers + + def perform! + ::Ci::Pipeline.transaction do + pipeline.save! + + @command.seeds_block&.call(pipeline) + + ::Ci::CreatePipelineStagesService + .new(project, current_user) + .execute(pipeline) + end + rescue ActiveRecord::RecordInvalid => e + error("Failed to persist the pipeline: #{e}") + end + + def break? + !pipeline.persisted? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb new file mode 100644 index 00000000000..02d81286f21 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Helpers + def branch_exists? + return @is_branch if defined?(@is_branch) + + @is_branch = project.repository.branch_exists?(pipeline.ref) + end + + def tag_exists? + return @is_tag if defined?(@is_tag) + + @is_tag = project.repository.tag_exists?(pipeline.ref) + end + + def error(message) + pipeline.errors.add(:base, message) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb new file mode 100644 index 00000000000..015f2988327 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -0,0 +1,36 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Sequence + def initialize(pipeline, command, sequence) + @pipeline = pipeline + @completed = [] + + @sequence = sequence.map do |chain| + chain.new(pipeline, command) + end + end + + def build! + @sequence.each do |step| + step.perform! + + break if step.break? + + @completed << step + end + + @pipeline.tap do + yield @pipeline, self if block_given? + end + end + + def complete? + @completed.size == @sequence.size + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb new file mode 100644 index 00000000000..9a72de87bab --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Skip < Chain::Base + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + + def perform! + if skipped? + @pipeline.skip if @command.save_incompleted + end + end + + def skipped? + !@command.ignore_skip_ci && commit_message_skips_ci? + end + + def break? + skipped? + end + + private + + def commit_message_skips_ci? + return false unless @pipeline.git_commit_message + + @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb new file mode 100644 index 00000000000..4913a604079 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -0,0 +1,54 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Abilities < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + + def perform! + unless project.builds_enabled? + return error('Pipelines are disabled!') + end + + unless allowed_to_trigger_pipeline? + if can?(current_user, :create_pipeline, project) + return error("Insufficient permissions for protected ref '#{pipeline.ref}'") + else + return error('Insufficient permissions to create a new pipeline') + end + end + end + + def break? + @pipeline.errors.any? + end + + def allowed_to_trigger_pipeline? + if current_user + allowed_to_create? + else # legacy triggers don't have a corresponding user + !project.protected_for?(@pipeline.ref) + end + end + + def allowed_to_create? + return unless can?(current_user, :create_pipeline, project) + + access = Gitlab::UserAccess.new(current_user, project: project) + + if branch_exists? + access.can_update_branch?(@pipeline.ref) + elsif tag_exists? + access.can_create_tag?(@pipeline.ref) + else + true # Allow it for now and we'll reject when we check ref existence + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb new file mode 100644 index 00000000000..075504bcce5 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -0,0 +1,35 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Config < Chain::Base + include Chain::Helpers + + def perform! + unless @pipeline.config_processor + unless @pipeline.ci_yaml_file + return error("Missing #{@pipeline.ci_yaml_file_path} file") + end + + if @command.save_incompleted && @pipeline.has_yaml_errors? + @pipeline.drop!(:config_error) + end + + return error(@pipeline.yaml_errors) + end + + unless @pipeline.has_stage_seeds? + return error('No stages / jobs for this pipeline.') + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb new file mode 100644 index 00000000000..70a4cfdbdea --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Repository < Chain::Base + include Chain::Helpers + + def perform! + unless branch_exists? || tag_exists? + return error('Reference not found') + end + + ## TODO, we check commit in the service, that is why + # there is no repository access here. + # + unless pipeline.sha + return error('Commit not found') + end + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb new file mode 100644 index 00000000000..469fc094cc8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -0,0 +1,143 @@ +module Gitlab + module Ci + module Pipeline + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module Duration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest + .where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb deleted file mode 100644 index 3208cc2bef6..00000000000 --- a/lib/gitlab/ci/pipeline_duration.rb +++ /dev/null @@ -1,141 +0,0 @@ -module Gitlab - module Ci - # # Introduction - total running time - # - # The problem this module is trying to solve is finding the total running - # time amongst all the jobs, excluding retries and pending (queue) time. - # We could reduce this problem down to finding the union of periods. - # - # So each job would be represented as a `Period`, which consists of - # `Period#first` as when the job started and `Period#last` as when the - # job was finished. A simple example here would be: - # - # * A (1, 3) - # * B (2, 4) - # * C (6, 7) - # - # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. - # C begins from 6, and ends to 7. Visually it could be viewed as: - # - # 0 1 2 3 4 5 6 7 - # AAAAAAA - # BBBBBBB - # CCCC - # - # The union of A, B, and C would be (1, 4) and (6, 7), therefore the - # total running time should be: - # - # (4 - 1) + (7 - 6) => 4 - # - # # The Algorithm - # - # The algorithm used here for union would be described as follow. - # First we make sure that all periods are sorted by `Period#first`. - # Then we try to merge periods by iterating through the first period - # to the last period. The goal would be merging all overlapped periods - # so that in the end all the periods are discrete. When all periods - # are discrete, we're free to just sum all the periods to get real - # running time. - # - # Here we begin from A, and compare it to B. We could find that - # before A ends, B already started. That is `B.first <= A.last` - # that is `2 <= 3` which means A and B are overlapping! - # - # When we found that two periods are overlapping, we would need to merge - # them into a new period and disregard the old periods. To make a new - # period, we take `A.first` as the new first because remember? we sorted - # them, so `A.first` must be smaller or equal to `B.first`. And we take - # `[A.last, B.last].max` as the new last because we want whoever ended - # later. This could be broken into two cases: - # - # 0 1 2 3 4 - # AAAAAAA - # BBBBBBB - # - # Or: - # - # 0 1 2 3 4 - # AAAAAAAAAA - # BBBB - # - # So that we need to take whoever ends later. Back to our example, - # after merging and discard A and B it could be visually viewed as: - # - # 0 1 2 3 4 5 6 7 - # DDDDDDDDDD - # CCCC - # - # Now we could go on and compare the newly created D and the old C. - # We could figure out that D and C are not overlapping by checking - # `C.first <= D.last` is `false`. Therefore we need to keep both C - # and D. The example would end here because there are no more jobs. - # - # After having the union of all periods, we just need to sum the length - # of all periods to get total time. - # - # (4 - 1) + (7 - 6) => 4 - # - # That is 4 is the answer in the example. - module PipelineDuration - extend self - - Period = Struct.new(:first, :last) do - def duration - last - first - end - end - - def from_pipeline(pipeline) - status = %w[success failed running canceled] - builds = pipeline.builds.latest - .where(status: status).where.not(started_at: nil).order(:started_at) - - from_builds(builds) - end - - def from_builds(builds) - now = Time.now - - periods = builds.map do |b| - Period.new(b.started_at, b.finished_at || now) - end - - from_periods(periods) - end - - # periods should be sorted by `first` - def from_periods(periods) - process_duration(process_periods(periods)) - end - - private - - def process_periods(periods) - return periods if periods.empty? - - periods.drop(1).inject([periods.first]) do |result, current| - previous = result.last - - if overlap?(previous, current) - result[-1] = merge(previous, current) - result - else - result << current - end - end - end - - def overlap?(previous, current) - current.first <= previous.last - end - - def merge(previous, current) - Period.new(previous.first, [previous.last, current.last].max) - end - - def process_duration(periods) - periods.sum(&:duration) - end - end - end -end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index e19aae35a81..bc97aa63b02 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -3,7 +3,9 @@ module Gitlab module Stage class Seed attr_reader :pipeline + delegate :project, to: :pipeline + delegate :size, to: :@jobs def initialize(pipeline, stage, jobs) @pipeline = pipeline diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 8ad3e57e59d..2d9166d6bdd 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -8,7 +8,7 @@ module Gitlab end def action_icon - 'icon_action_cancel' + 'cancel' end def action_path diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index e42d3574357..d71e63e73eb 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -8,7 +8,7 @@ module Gitlab end def icon - 'icon_status_warning' + 'warning' end def group diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c7726543599..b7b45466d3b 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -12,7 +12,7 @@ module Gitlab end def action_icon - 'icon_action_play' + 'play' end def action_title diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 8c8fdc56d75..44ffe783e50 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -8,7 +8,7 @@ module Gitlab end def action_icon - 'icon_action_retry' + 'retry' end def action_title diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index d464738deaf..46e730797e4 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -12,7 +12,7 @@ module Gitlab end def action_icon - 'icon_action_stop' + 'stop' end def action_title diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index e5fdc1f8136..e6195a60d4f 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_canceled' + 'status_canceled' end def favicon diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index d188bd286a6..846f00b83dd 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_created' + 'status_created' end def favicon diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 38e45714c22..27ce85bd3ed 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_failed' + 'status_failed' end def favicon diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index a4a7edadac9..fc387e2fd25 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_manual' + 'status_manual' end def favicon diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index 5164260b861..6780780db32 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_pending' + 'status_pending' end def favicon diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index 993937e98ca..ee13905e46d 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_running' + 'status_running' end def favicon diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 0c942920b02..0dbdc4de426 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_skipped' + 'status_skipped' end def favicon diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index d7af98857b0..731013ec017 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -11,7 +11,7 @@ module Gitlab end def icon - 'icon_status_success' + 'status_success' end def favicon diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 4d7d82e04cf..32b4cf43e48 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -15,7 +15,7 @@ module Gitlab end def icon - 'icon_status_warning' + 'status_warning' end def group diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 5b835bb669a..baf55b1fa07 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -27,6 +27,12 @@ module Gitlab end end + def extract_sections + read do |stream| + stream.extract_sections + end + end + def set(data) write do |stream| data = job.hide_secrets(data) diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb new file mode 100644 index 00000000000..9bb0166c9e3 --- /dev/null +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -0,0 +1,97 @@ +module Gitlab + module Ci + class Trace + class SectionParser + def initialize(lines) + @lines = lines + end + + def parse! + @markers = {} + + @lines.each do |line, pos| + parse_line(line, pos) + end + end + + def sections + sanitize_markers.map do |name, markers| + start_, end_ = markers + + { + name: name, + byte_start: start_[:marker], + byte_end: end_[:marker], + date_start: start_[:timestamp], + date_end: end_[:timestamp] + } + end + end + + private + + def parse_line(line, line_start_position) + s = StringScanner.new(line) + until s.eos? + find_next_marker(s) do |scanner| + marker_begins_at = line_start_position + scanner.pointer + + if scanner.scan(Gitlab::Regex.build_trace_section_regex) + marker_ends_at = line_start_position + scanner.pointer + handle_line(scanner[1], scanner[2].to_i, scanner[3], marker_begins_at, marker_ends_at) + true + else + false + end + end + end + end + + def sanitize_markers + @markers.select do |_, markers| + markers.size == 2 && markers[0][:action] == :start && markers[1][:action] == :end + end + end + + def handle_line(action, time, name, marker_start, marker_end) + action = action.to_sym + timestamp = Time.at(time).utc + marker = if action == :start + marker_end + else + marker_start + end + + @markers[name] ||= [] + @markers[name] << { + name: name, + action: action, + timestamp: timestamp, + marker: marker + } + end + + def beginning_of_section_regex + @beginning_of_section_regex ||= /section_/.freeze + end + + def find_next_marker(s) + beginning_of_section_len = 8 + maybe_marker = s.exist?(beginning_of_section_regex) + + if maybe_marker.nil? + s.terminate + else + # repositioning at the beginning of the match + s.pos += maybe_marker - beginning_of_section_len + if block_given? + good_marker = yield(s) + # if not a good marker: Consuming the matched beginning_of_section_regex + s.pos += beginning_of_section_len unless good_marker + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 8503ecf8700..d52194f688b 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -56,13 +56,13 @@ module Gitlab end def html_with_state(state = nil) - ::Ci::Ansi2html.convert(stream, state) + ::Gitlab::Ci::Ansi2html.convert(stream, state) end def html(last_lines: nil) text = raw(last_lines: last_lines) buffer = StringIO.new(text) - ::Ci::Ansi2html.convert(buffer).html + ::Gitlab::Ci::Ansi2html.convert(buffer).html end def extract_coverage(regex) @@ -90,8 +90,25 @@ module Gitlab # so we just silently ignore error for now end + def extract_sections + return [] unless valid? + + lines = to_enum(:each_line_with_pos) + parser = SectionParser.new(lines) + + parser.parse! + parser.sections + end + private + def each_line_with_pos + stream.seek(0, IO::SEEK_SET) + stream.each_line do |line| + yield [line, stream.pos - line.bytesize] + end + end + def read_last_lines(limit) to_enum(:reverse_line).first(limit).reverse.join end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb new file mode 100644 index 00000000000..0bd78b03448 --- /dev/null +++ b/lib/gitlab/ci/yaml_processor.rb @@ -0,0 +1,189 @@ +module Gitlab + module Ci + class YamlProcessor + ValidationError = Class.new(StandardError) + + include Gitlab::Ci::Config::Entry::LegacyValidationHelpers + + attr_reader :cache, :stages, :jobs + + def initialize(config) + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + + initial_parsing + rescue Gitlab::Ci::Config::Loader::FormatError => e + raise ValidationError, e.message + end + + def builds + @jobs.map do |name, _| + build_attributes(name) + end + end + + def build_attributes(name) + job = @jobs[name.to_sym] || {} + + { stage_idx: @stages.index(job[:stage]), + stage: job[:stage], + commands: job[:commands], + tag_list: job[:tags] || [], + name: job[:name].to_s, + allow_failure: job[:ignore], + when: job[:when] || 'on_success', + environment: job[:environment_name], + coverage_regex: job[:coverage], + yaml_variables: yaml_variables(name), + options: { + image: job[:image], + services: job[:services], + artifacts: job[:artifacts], + cache: job[:cache], + dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], + after_script: job[:after_script], + environment: job[:environment], + retry: job[:retry] + }.compact } + end + + def pipeline_stage_builds(stage, pipeline) + selected_jobs = @jobs.select do |_, job| + next unless job[:stage] == stage + + only_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:only, {})) + except_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:except, {})) + + only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && + except_specs.none? { |spec| spec.satisfied_by?(pipeline) } + end + + selected_jobs.map { |_, job| build_attributes(job[:name]) } + end + + def stage_seeds(pipeline) + seeds = @stages.uniq.map do |stage| + builds = pipeline_stage_builds(stage, pipeline) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Gitlab::Ci::YamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + + private + + def initial_parsing + ## + # Global config + # + @before_script = @ci_config.before_script + @image = @ci_config.image + @after_script = @ci_config.after_script + @services = @ci_config.services + @variables = @ci_config.variables + @stages = @ci_config.stages + @cache = @ci_config.cache + + ## + # Jobs + # + @jobs = @ci_config.jobs + + @jobs.each do |name, job| + # logical validation for job + + validate_job_stage!(name, job) + validate_job_dependencies!(name, job) + validate_job_environment!(name, job) + end + end + + def yaml_variables(name) + variables = (@variables || {}) + .merge(job_variables(name)) + + variables.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def job_variables(name) + job = @jobs[name.to_sym] + return {} unless job + + job[:variables] || {} + end + + def validate_job_stage!(name, job) + return unless job[:stage] + + unless job[:stage].is_a?(String) && job[:stage].in?(@stages) + raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" + end + end + + def validate_job_dependencies!(name, job) + return unless job[:dependencies] + + stage_index = @stages.index(job[:stage]) + + job[:dependencies].each do |dependency| + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] + + unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" + end + end + end + + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + end + end +end |