Merge branch '5836-move-lib-ci-into-gitlab-namespace' into 'master'

Resolve "Move `lib/ci` to `lib/gitlab/ci`"

Closes #5836

See merge request !14078

13 files changed, 732 insertions, 722 deletions

diff --git a/lib/api/lint.rb b/lib/api/lint.rb
index ae43a4a3237..d202eaa4c49 100644
--- a/lib/api/lint.rb
+++ b/lib/api/lint.rb
@@ -6,7 +6,7 @@ module API
         requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
       end
       post '/lint' do
-        error = Ci::GitlabCiYamlProcessor.validation_message(params[:content])
+        error = Gitlab::Ci::YamlProcessor.validation_message(params[:content])
 
         status 200
 
diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
deleted file mode 100644
index b9e9f9f7f4a..00000000000
--- a/lib/ci/ansi2html.rb
+++ /dev/null
@@ -1,331 +0,0 @@
-# ANSI color library
-#
-# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code
-module Ci
-  module Ansi2html
-    # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
-    COLOR = {
-      0 => 'black', # not that this is gray in the intense color table
-      1 => 'red',
-      2 => 'green',
-      3 => 'yellow',
-      4 => 'blue',
-      5 => 'magenta',
-      6 => 'cyan',
-      7 => 'white', # not that this is gray in the dark (aka default) color table
-    }.freeze
-
-    STYLE_SWITCHES = {
-      bold:       0x01,
-      italic:     0x02,
-      underline:  0x04,
-      conceal:    0x08,
-      cross:      0x10
-    }.freeze
-
-    def self.convert(ansi, state = nil)
-      Converter.new.convert(ansi, state)
-    end
-
-    class Converter
-      def on_0(s) reset()                            end
-
-      def on_1(s) enable(STYLE_SWITCHES[:bold])      end
-
-      def on_3(s) enable(STYLE_SWITCHES[:italic])    end
-
-      def on_4(s) enable(STYLE_SWITCHES[:underline]) end
-
-      def on_8(s) enable(STYLE_SWITCHES[:conceal])   end
-
-      def on_9(s) enable(STYLE_SWITCHES[:cross])     end
-
-      def on_21(s) disable(STYLE_SWITCHES[:bold])      end
-
-      def on_22(s) disable(STYLE_SWITCHES[:bold])      end
-
-      def on_23(s) disable(STYLE_SWITCHES[:italic])    end
-
-      def on_24(s) disable(STYLE_SWITCHES[:underline]) end
-
-      def on_28(s) disable(STYLE_SWITCHES[:conceal])   end
-
-      def on_29(s) disable(STYLE_SWITCHES[:cross])     end
-
-      def on_30(s) set_fg_color(0) end
-
-      def on_31(s) set_fg_color(1) end
-
-      def on_32(s) set_fg_color(2) end
-
-      def on_33(s) set_fg_color(3) end
-
-      def on_34(s) set_fg_color(4) end
-
-      def on_35(s) set_fg_color(5) end
-
-      def on_36(s) set_fg_color(6) end
-
-      def on_37(s) set_fg_color(7) end
-
-      def on_38(s) set_fg_color_256(s) end
-
-      def on_39(s) set_fg_color(9) end
-
-      def on_40(s) set_bg_color(0) end
-
-      def on_41(s) set_bg_color(1) end
-
-      def on_42(s) set_bg_color(2) end
-
-      def on_43(s) set_bg_color(3) end
-
-      def on_44(s) set_bg_color(4) end
-
-      def on_45(s) set_bg_color(5) end
-
-      def on_46(s) set_bg_color(6) end
-
-      def on_47(s) set_bg_color(7) end
-
-      def on_48(s) set_bg_color_256(s) end
-
-      def on_49(s) set_bg_color(9) end
-
-      def on_90(s) set_fg_color(0, 'l') end
-
-      def on_91(s) set_fg_color(1, 'l') end
-
-      def on_92(s) set_fg_color(2, 'l') end
-
-      def on_93(s) set_fg_color(3, 'l') end
-
-      def on_94(s) set_fg_color(4, 'l') end
-
-      def on_95(s) set_fg_color(5, 'l') end
-
-      def on_96(s) set_fg_color(6, 'l') end
-
-      def on_97(s) set_fg_color(7, 'l') end
-
-      def on_99(s) set_fg_color(9, 'l') end
-
-      def on_100(s) set_bg_color(0, 'l') end
-
-      def on_101(s) set_bg_color(1, 'l') end
-
-      def on_102(s) set_bg_color(2, 'l') end
-
-      def on_103(s) set_bg_color(3, 'l') end
-
-      def on_104(s) set_bg_color(4, 'l') end
-
-      def on_105(s) set_bg_color(5, 'l') end
-
-      def on_106(s) set_bg_color(6, 'l') end
-
-      def on_107(s) set_bg_color(7, 'l') end
-
-      def on_109(s) set_bg_color(9, 'l') end
-
-      attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask
-
-      STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
-
-      def convert(stream, new_state)
-        reset_state
-        restore_state(new_state, stream) if new_state.present?
-
-        append = false
-        truncated = false
-
-        cur_offset = stream.tell
-        if cur_offset > @offset
-          @offset = cur_offset
-          truncated = true
-        else
-          stream.seek(@offset)
-          append = @offset > 0
-        end
-        start_offset = @offset
-
-        open_new_tag
-
-        stream.each_line do |line|
-          s = StringScanner.new(line)
-          until s.eos?
-            if s.scan(/\e([@-_])(.*?)([@-~])/)
-              handle_sequence(s)
-            elsif s.scan(/\e(([@-_])(.*?)?)?$/)
-              break
-            elsif s.scan(/</)
-              @out << '&lt;'
-            elsif s.scan(/\r?\n/)
-              @out << '<br>'
-            else
-              @out << s.scan(/./m)
-            end
-            @offset += s.matched_size
-          end
-        end
-
-        close_open_tags()
-
-        OpenStruct.new(
-          html: @out.force_encoding(Encoding.default_external),
-          state: state,
-          append: append,
-          truncated: truncated,
-          offset: start_offset,
-          size: stream.tell - start_offset,
-          total: stream.size
-        )
-      end
-
-      def handle_sequence(s)
-        indicator = s[1]
-        commands = s[2].split ';'
-        terminator = s[3]
-
-        # We are only interested in color and text style changes - triggered by
-        # sequences starting with '\e[' and ending with 'm'. Any other control
-        # sequence gets stripped (including stuff like "delete last line")
-        return unless indicator == '[' && terminator == 'm'
-
-        close_open_tags()
-
-        if commands.empty?()
-          reset()
-          return
-        end
-
-        evaluate_command_stack(commands)
-
-        open_new_tag
-      end
-
-      def evaluate_command_stack(stack)
-        return unless command = stack.shift()
-
-        if self.respond_to?("on_#{command}", true)
-          self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend
-        end
-
-        evaluate_command_stack(stack)
-      end
-
-      def open_new_tag
-        css_classes = []
-
-        unless @fg_color.nil?
-          fg_color = @fg_color
-          # Most terminals show bold colored text in the light color variant
-          # Let's mimic that here
-          if @style_mask & STYLE_SWITCHES[:bold] != 0
-            fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1')
-          end
-          css_classes << fg_color
-        end
-        css_classes << @bg_color unless @bg_color.nil?
-
-        STYLE_SWITCHES.each do |css_class, flag|
-          css_classes << "term-#{css_class}" if @style_mask & flag != 0
-        end
-
-        return if css_classes.empty?
-
-        @out << %{<span class="#{css_classes.join(' ')}">}
-        @n_open_tags += 1
-      end
-
-      def close_open_tags
-        while @n_open_tags > 0
-          @out << %{</span>}
-          @n_open_tags -= 1
-        end
-      end
-
-      def reset_state
-        @offset = 0
-        @n_open_tags = 0
-        @out = ''
-        reset
-      end
-
-      def state
-        state = STATE_PARAMS.inject({}) do |h, param|
-          h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend
-          h
-        end
-        Base64.urlsafe_encode64(state.to_json)
-      end
-
-      def restore_state(new_state, stream)
-        state = Base64.urlsafe_decode64(new_state)
-        state = JSON.parse(state, symbolize_names: true)
-        return if state[:offset].to_i > stream.size
-
-        STATE_PARAMS.each do |param|
-          send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend
-        end
-      end
-
-      def reset
-        @fg_color = nil
-        @bg_color = nil
-        @style_mask = 0
-      end
-
-      def enable(flag)
-        @style_mask |= flag
-      end
-
-      def disable(flag)
-        @style_mask &= ~flag
-      end
-
-      def set_fg_color(color_index, prefix = nil)
-        @fg_color = get_term_color_class(color_index, ["fg", prefix])
-      end
-
-      def set_bg_color(color_index, prefix = nil)
-        @bg_color = get_term_color_class(color_index, ["bg", prefix])
-      end
-
-      def get_term_color_class(color_index, prefix)
-        color_name = COLOR[color_index]
-        return nil if color_name.nil?
-
-        get_color_class(["term", prefix, color_name])
-      end
-
-      def set_fg_color_256(command_stack)
-        css_class = get_xterm_color_class(command_stack, "fg")
-        @fg_color = css_class unless css_class.nil?
-      end
-
-      def set_bg_color_256(command_stack)
-        css_class = get_xterm_color_class(command_stack, "bg")
-        @bg_color = css_class unless css_class.nil?
-      end
-
-      def get_xterm_color_class(command_stack, prefix)
-        # the 38 and 48 commands have to be followed by "5" and the color index
-        return unless command_stack.length >= 2
-        return unless command_stack[0] == "5"
-
-        command_stack.shift() # ignore the "5" command
-        color_index = command_stack.shift().to_i
-
-        return unless color_index >= 0
-        return unless color_index <= 255
-
-        get_color_class(["xterm", prefix, color_index])
-      end
-
-      def get_color_class(segments)
-        [segments].flatten.compact.join('-')
-      end
-    end
-  end
-end
diff --git a/lib/ci/assets/.gitkeep b/lib/ci/assets/.gitkeep
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/lib/ci/assets/.gitkeep
+++ /dev/null
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
deleted file mode 100644
index 76a69bf8a83..00000000000
--- a/lib/ci/charts.rb
+++ /dev/null
@@ -1,116 +0,0 @@
-module Ci
-  module Charts
-    module DailyInterval
-      def grouped_count(query)
-        query
-          .group("DATE(#{Ci::Pipeline.table_name}.created_at)")
-          .count(:created_at)
-          .transform_keys { |date| date.strftime(@format) }
-      end
-
-      def interval_step
-        @interval_step ||= 1.day
-      end
-    end
-
-    module MonthlyInterval
-      def grouped_count(query)
-        if Gitlab::Database.postgresql?
-          query
-            .group("to_char(#{Ci::Pipeline.table_name}.created_at, '01 Month YYYY')")
-            .count(:created_at)
-            .transform_keys(&:squish)
-        else
-          query
-            .group("DATE_FORMAT(#{Ci::Pipeline.table_name}.created_at, '01 %M %Y')")
-            .count(:created_at)
-        end
-      end
-
-      def interval_step
-        @interval_step ||= 1.month
-      end
-    end
-
-    class Chart
-      attr_reader :labels, :total, :success, :project, :pipeline_times
-
-      def initialize(project)
-        @labels = []
-        @total = []
-        @success = []
-        @pipeline_times = []
-        @project = project
-
-        collect
-      end
-
-      def collect
-        query = project.pipelines
-          .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection
-
-        totals_count  = grouped_count(query)
-        success_count = grouped_count(query.success)
-
-        current = @from
-        while current < @to
-          label = current.strftime(@format)
-
-          @labels  << label
-          @total   << (totals_count[label] || 0)
-          @success << (success_count[label] || 0)
-
-          current += interval_step
-        end
-      end
-    end
-
-    class YearChart < Chart
-      include MonthlyInterval
-
-      def initialize(*)
-        @to     = Date.today.end_of_month
-        @from   = @to.years_ago(1).beginning_of_month
-        @format = '%d %B %Y'
-
-        super
-      end
-    end
-
-    class MonthChart < Chart
-      include DailyInterval
-
-      def initialize(*)
-        @to     = Date.today
-        @from   = @to - 30.days
-        @format = '%d %B'
-
-        super
-      end
-    end
-
-    class WeekChart < Chart
-      include DailyInterval
-
-      def initialize(*)
-        @to     = Date.today
-        @from   = @to - 7.days
-        @format = '%d %B'
-
-        super
-      end
-    end
-
-    class PipelineTime < Chart
-      def collect
-        commits = project.pipelines.last(30)
-
-        commits.each do |commit|
-          @labels << commit.short_sha
-          duration = commit.duration || 0
-          @pipeline_times << (duration / 60)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
deleted file mode 100644
index 62b44389b15..00000000000
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ /dev/null
@@ -1,251 +0,0 @@
-module Ci
-  class GitlabCiYamlProcessor
-    ValidationError = Class.new(StandardError)
-
-    include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
-
-    attr_reader :path, :cache, :stages, :jobs
-
-    def initialize(config, path = nil)
-      @ci_config = Gitlab::Ci::Config.new(config)
-      @config = @ci_config.to_hash
-      @path = path
-
-      unless @ci_config.valid?
-        raise ValidationError, @ci_config.errors.first
-      end
-
-      initial_parsing
-    rescue Gitlab::Ci::Config::Loader::FormatError => e
-      raise ValidationError, e.message
-    end
-
-    def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
-      jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
-        build_attributes(name)
-      end
-    end
-
-    def builds
-      @jobs.map do |name, _|
-        build_attributes(name)
-      end
-    end
-
-    def stage_seeds(pipeline)
-      seeds = @stages.uniq.map do |stage|
-        builds = pipeline_stage_builds(stage, pipeline)
-
-        Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
-      end
-
-      seeds.compact
-    end
-
-    def build_attributes(name)
-      job = @jobs[name.to_sym] || {}
-
-      { stage_idx: @stages.index(job[:stage]),
-        stage: job[:stage],
-        commands: job[:commands],
-        tag_list: job[:tags] || [],
-        name: job[:name].to_s,
-        allow_failure: job[:ignore],
-        when: job[:when] || 'on_success',
-        environment: job[:environment_name],
-        coverage_regex: job[:coverage],
-        yaml_variables: yaml_variables(name),
-        options: {
-          image: job[:image],
-          services: job[:services],
-          artifacts: job[:artifacts],
-          cache: job[:cache],
-          dependencies: job[:dependencies],
-          before_script: job[:before_script],
-          script: job[:script],
-          after_script: job[:after_script],
-          environment: job[:environment],
-          retry: job[:retry]
-        }.compact }
-    end
-
-    def self.validation_message(content)
-      return 'Please provide content of .gitlab-ci.yml' if content.blank?
-
-      begin
-        Ci::GitlabCiYamlProcessor.new(content)
-        nil
-      rescue ValidationError, Psych::SyntaxError => e
-        e.message
-      end
-    end
-
-    private
-
-    def pipeline_stage_builds(stage, pipeline)
-      builds = builds_for_stage_and_ref(
-        stage, pipeline.ref, pipeline.tag?, pipeline.source)
-
-      builds.select do |build|
-        job = @jobs[build.fetch(:name).to_sym]
-        has_kubernetes = pipeline.has_kubernetes_active?
-        only_kubernetes = job.dig(:only, :kubernetes)
-        except_kubernetes = job.dig(:except, :kubernetes)
-
-        [!only_kubernetes && !except_kubernetes,
-         only_kubernetes && has_kubernetes,
-         except_kubernetes && !has_kubernetes].any?
-      end
-    end
-
-    def jobs_for_ref(ref, tag = false, source = nil)
-      @jobs.select do |_, job|
-        process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source)
-      end
-    end
-
-    def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
-      jobs_for_ref(ref, tag, source).select do |_, job|
-        job[:stage] == stage
-      end
-    end
-
-    def initial_parsing
-      ##
-      # Global config
-      #
-      @before_script = @ci_config.before_script
-      @image = @ci_config.image
-      @after_script = @ci_config.after_script
-      @services = @ci_config.services
-      @variables = @ci_config.variables
-      @stages = @ci_config.stages
-      @cache = @ci_config.cache
-
-      ##
-      # Jobs
-      #
-      @jobs = @ci_config.jobs
-
-      @jobs.each do |name, job|
-        # logical validation for job
-
-        validate_job_stage!(name, job)
-        validate_job_dependencies!(name, job)
-        validate_job_environment!(name, job)
-      end
-    end
-
-    def yaml_variables(name)
-      variables = (@variables || {})
-        .merge(job_variables(name))
-
-      variables.map do |key, value|
-        { key: key.to_s, value: value, public: true }
-      end
-    end
-
-    def job_variables(name)
-      job = @jobs[name.to_sym]
-      return {} unless job
-
-      job[:variables] || {}
-    end
-
-    def validate_job_stage!(name, job)
-      return unless job[:stage]
-
-      unless job[:stage].is_a?(String) && job[:stage].in?(@stages)
-        raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}"
-      end
-    end
-
-    def validate_job_dependencies!(name, job)
-      return unless job[:dependencies]
-
-      stage_index = @stages.index(job[:stage])
-
-      job[:dependencies].each do |dependency|
-        raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym]
-
-        unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index
-          raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages"
-        end
-      end
-    end
-
-    def validate_job_environment!(name, job)
-      return unless job[:environment]
-      return unless job[:environment].is_a?(Hash)
-
-      environment = job[:environment]
-      validate_on_stop_job!(name, environment, environment[:on_stop])
-    end
-
-    def validate_on_stop_job!(name, environment, on_stop)
-      return unless on_stop
-
-      on_stop_job = @jobs[on_stop.to_sym]
-      unless on_stop_job
-        raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined"
-      end
-
-      unless on_stop_job[:environment]
-        raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined"
-      end
-
-      unless on_stop_job[:environment][:name] == environment[:name]
-        raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name"
-      end
-
-      unless on_stop_job[:environment][:action] == 'stop'
-        raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined"
-      end
-    end
-
-    def process?(only_params, except_params, ref, tag, source)
-      if only_params.present?
-        return false unless matching?(only_params, ref, tag, source)
-      end
-
-      if except_params.present?
-        return false if matching?(except_params, ref, tag, source)
-      end
-
-      true
-    end
-
-    def matching?(patterns, ref, tag, source)
-      patterns.any? do |pattern|
-        pattern, path = pattern.split('@', 2)
-        matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
-      end
-    end
-
-    def matches_path?(path)
-      return true unless path
-
-      path == self.path
-    end
-
-    def matches_pattern?(pattern, ref, tag, source)
-      return true if tag && pattern == 'tags'
-      return true if !tag && pattern == 'branches'
-      return true if source_to_pattern(source) == pattern
-
-      if pattern.first == "/" && pattern.last == "/"
-        Regexp.new(pattern[1...-1]) =~ ref
-      else
-        pattern == ref
-      end
-    end
-
-    def source_to_pattern(source)
-      if %w[api external web].include?(source)
-        source
-      else
-        source&.pluralize
-      end
-    end
-  end
-end
diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb
deleted file mode 100644
index 997377abc55..00000000000
--- a/lib/ci/mask_secret.rb
+++ /dev/null
@@ -1,10 +0,0 @@
-module Ci::MaskSecret
-  class << self
-    def mask!(value, token)
-      return value unless value.present? && token.present?
-
-      value.gsub!(token, 'x' * token.length)
-      value
-    end
-  end
-end
diff --git a/lib/ci/model.rb b/lib/ci/model.rb
deleted file mode 100644
index c42a0ad36db..00000000000
--- a/lib/ci/model.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-module Ci
-  module Model
-    def table_name_prefix
-      "ci_"
-    end
-
-    def model_name
-      @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last)
-    end
-  end
-end
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
new file mode 100644
index 00000000000..ad78ae244b2
--- /dev/null
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -0,0 +1,333 @@
+# ANSI color library
+#
+# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code
+module Gitlab
+  module Ci
+    module Ansi2html
+      # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
+      COLOR = {
+        0 => 'black', # not that this is gray in the intense color table
+        1 => 'red',
+        2 => 'green',
+        3 => 'yellow',
+        4 => 'blue',
+        5 => 'magenta',
+        6 => 'cyan',
+        7 => 'white', # not that this is gray in the dark (aka default) color table
+      }.freeze
+
+      STYLE_SWITCHES = {
+        bold:       0x01,
+        italic:     0x02,
+        underline:  0x04,
+        conceal:    0x08,
+        cross:      0x10
+      }.freeze
+
+      def self.convert(ansi, state = nil)
+        Converter.new.convert(ansi, state)
+      end
+
+      class Converter
+        def on_0(s) reset()                            end
+
+        def on_1(s) enable(STYLE_SWITCHES[:bold])      end
+
+        def on_3(s) enable(STYLE_SWITCHES[:italic])    end
+
+        def on_4(s) enable(STYLE_SWITCHES[:underline]) end
+
+        def on_8(s) enable(STYLE_SWITCHES[:conceal])   end
+
+        def on_9(s) enable(STYLE_SWITCHES[:cross])     end
+
+        def on_21(s) disable(STYLE_SWITCHES[:bold])      end
+
+        def on_22(s) disable(STYLE_SWITCHES[:bold])      end
+
+        def on_23(s) disable(STYLE_SWITCHES[:italic])    end
+
+        def on_24(s) disable(STYLE_SWITCHES[:underline]) end
+
+        def on_28(s) disable(STYLE_SWITCHES[:conceal])   end
+
+        def on_29(s) disable(STYLE_SWITCHES[:cross])     end
+
+        def on_30(s) set_fg_color(0) end
+
+        def on_31(s) set_fg_color(1) end
+
+        def on_32(s) set_fg_color(2) end
+
+        def on_33(s) set_fg_color(3) end
+
+        def on_34(s) set_fg_color(4) end
+
+        def on_35(s) set_fg_color(5) end
+
+        def on_36(s) set_fg_color(6) end
+
+        def on_37(s) set_fg_color(7) end
+
+        def on_38(s) set_fg_color_256(s) end
+
+        def on_39(s) set_fg_color(9) end
+
+        def on_40(s) set_bg_color(0) end
+
+        def on_41(s) set_bg_color(1) end
+
+        def on_42(s) set_bg_color(2) end
+
+        def on_43(s) set_bg_color(3) end
+
+        def on_44(s) set_bg_color(4) end
+
+        def on_45(s) set_bg_color(5) end
+
+        def on_46(s) set_bg_color(6) end
+
+        def on_47(s) set_bg_color(7) end
+
+        def on_48(s) set_bg_color_256(s) end
+
+        def on_49(s) set_bg_color(9) end
+
+        def on_90(s) set_fg_color(0, 'l') end
+
+        def on_91(s) set_fg_color(1, 'l') end
+
+        def on_92(s) set_fg_color(2, 'l') end
+
+        def on_93(s) set_fg_color(3, 'l') end
+
+        def on_94(s) set_fg_color(4, 'l') end
+
+        def on_95(s) set_fg_color(5, 'l') end
+
+        def on_96(s) set_fg_color(6, 'l') end
+
+        def on_97(s) set_fg_color(7, 'l') end
+
+        def on_99(s) set_fg_color(9, 'l') end
+
+        def on_100(s) set_bg_color(0, 'l') end
+
+        def on_101(s) set_bg_color(1, 'l') end
+
+        def on_102(s) set_bg_color(2, 'l') end
+
+        def on_103(s) set_bg_color(3, 'l') end
+
+        def on_104(s) set_bg_color(4, 'l') end
+
+        def on_105(s) set_bg_color(5, 'l') end
+
+        def on_106(s) set_bg_color(6, 'l') end
+
+        def on_107(s) set_bg_color(7, 'l') end
+
+        def on_109(s) set_bg_color(9, 'l') end
+
+        attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask
+
+        STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
+
+        def convert(stream, new_state)
+          reset_state
+          restore_state(new_state, stream) if new_state.present?
+
+          append = false
+          truncated = false
+
+          cur_offset = stream.tell
+          if cur_offset > @offset
+            @offset = cur_offset
+            truncated = true
+          else
+            stream.seek(@offset)
+            append = @offset > 0
+          end
+          start_offset = @offset
+
+          open_new_tag
+
+          stream.each_line do |line|
+            s = StringScanner.new(line)
+            until s.eos?
+              if s.scan(/\e([@-_])(.*?)([@-~])/)
+                handle_sequence(s)
+              elsif s.scan(/\e(([@-_])(.*?)?)?$/)
+                break
+              elsif s.scan(/</)
+                @out << '&lt;'
+              elsif s.scan(/\r?\n/)
+                @out << '<br>'
+              else
+                @out << s.scan(/./m)
+              end
+              @offset += s.matched_size
+            end
+          end
+
+          close_open_tags()
+
+          OpenStruct.new(
+            html: @out.force_encoding(Encoding.default_external),
+            state: state,
+            append: append,
+            truncated: truncated,
+            offset: start_offset,
+            size: stream.tell - start_offset,
+            total: stream.size
+          )
+        end
+
+        def handle_sequence(s)
+          indicator = s[1]
+          commands = s[2].split ';'
+          terminator = s[3]
+
+          # We are only interested in color and text style changes - triggered by
+          # sequences starting with '\e[' and ending with 'm'. Any other control
+          # sequence gets stripped (including stuff like "delete last line")
+          return unless indicator == '[' && terminator == 'm'
+
+          close_open_tags()
+
+          if commands.empty?()
+            reset()
+            return
+          end
+
+          evaluate_command_stack(commands)
+
+          open_new_tag
+        end
+
+        def evaluate_command_stack(stack)
+          return unless command = stack.shift()
+
+          if self.respond_to?("on_#{command}", true)
+            self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend
+          end
+
+          evaluate_command_stack(stack)
+        end
+
+        def open_new_tag
+          css_classes = []
+
+          unless @fg_color.nil?
+            fg_color = @fg_color
+            # Most terminals show bold colored text in the light color variant
+            # Let's mimic that here
+            if @style_mask & STYLE_SWITCHES[:bold] != 0
+              fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1')
+            end
+            css_classes << fg_color
+          end
+          css_classes << @bg_color unless @bg_color.nil?
+
+          STYLE_SWITCHES.each do |css_class, flag|
+            css_classes << "term-#{css_class}" if @style_mask & flag != 0
+          end
+
+          return if css_classes.empty?
+
+          @out << %{<span class="#{css_classes.join(' ')}">}
+          @n_open_tags += 1
+        end
+
+        def close_open_tags
+          while @n_open_tags > 0
+            @out << %{</span>}
+            @n_open_tags -= 1
+          end
+        end
+
+        def reset_state
+          @offset = 0
+          @n_open_tags = 0
+          @out = ''
+          reset
+        end
+
+        def state
+          state = STATE_PARAMS.inject({}) do |h, param|
+            h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend
+            h
+          end
+          Base64.urlsafe_encode64(state.to_json)
+        end
+
+        def restore_state(new_state, stream)
+          state = Base64.urlsafe_decode64(new_state)
+          state = JSON.parse(state, symbolize_names: true)
+          return if state[:offset].to_i > stream.size
+
+          STATE_PARAMS.each do |param|
+            send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        def reset
+          @fg_color = nil
+          @bg_color = nil
+          @style_mask = 0
+        end
+
+        def enable(flag)
+          @style_mask |= flag
+        end
+
+        def disable(flag)
+          @style_mask &= ~flag
+        end
+
+        def set_fg_color(color_index, prefix = nil)
+          @fg_color = get_term_color_class(color_index, ["fg", prefix])
+        end
+
+        def set_bg_color(color_index, prefix = nil)
+          @bg_color = get_term_color_class(color_index, ["bg", prefix])
+        end
+
+        def get_term_color_class(color_index, prefix)
+          color_name = COLOR[color_index]
+          return nil if color_name.nil?
+
+          get_color_class(["term", prefix, color_name])
+        end
+
+        def set_fg_color_256(command_stack)
+          css_class = get_xterm_color_class(command_stack, "fg")
+          @fg_color = css_class unless css_class.nil?
+        end
+
+        def set_bg_color_256(command_stack)
+          css_class = get_xterm_color_class(command_stack, "bg")
+          @bg_color = css_class unless css_class.nil?
+        end
+
+        def get_xterm_color_class(command_stack, prefix)
+          # the 38 and 48 commands have to be followed by "5" and the color index
+          return unless command_stack.length >= 2
+          return unless command_stack[0] == "5"
+
+          command_stack.shift() # ignore the "5" command
+          color_index = command_stack.shift().to_i
+
+          return unless color_index >= 0
+          return unless color_index <= 255
+
+          get_color_class(["xterm", prefix, color_index])
+        end
+
+        def get_color_class(segments)
+          [segments].flatten.compact.join('-')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb
new file mode 100644
index 00000000000..7df7b542d91
--- /dev/null
+++ b/lib/gitlab/ci/charts.rb
@@ -0,0 +1,118 @@
+module Gitlab
+  module Ci
+    module Charts
+      module DailyInterval
+        def grouped_count(query)
+          query
+            .group("DATE(#{::Ci::Pipeline.table_name}.created_at)")
+            .count(:created_at)
+            .transform_keys { |date| date.strftime(@format) }
+        end
+
+        def interval_step
+          @interval_step ||= 1.day
+        end
+      end
+
+      module MonthlyInterval
+        def grouped_count(query)
+          if Gitlab::Database.postgresql?
+            query
+              .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')")
+              .count(:created_at)
+              .transform_keys(&:squish)
+          else
+            query
+              .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')")
+              .count(:created_at)
+          end
+        end
+
+        def interval_step
+          @interval_step ||= 1.month
+        end
+      end
+
+      class Chart
+        attr_reader :labels, :total, :success, :project, :pipeline_times
+
+        def initialize(project)
+          @labels = []
+          @total = []
+          @success = []
+          @pipeline_times = []
+          @project = project
+
+          collect
+        end
+
+        def collect
+          query = project.pipelines
+            .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection
+
+          totals_count  = grouped_count(query)
+          success_count = grouped_count(query.success)
+
+          current = @from
+          while current < @to
+            label = current.strftime(@format)
+
+            @labels  << label
+            @total   << (totals_count[label] || 0)
+            @success << (success_count[label] || 0)
+
+            current += interval_step
+          end
+        end
+      end
+
+      class YearChart < Chart
+        include MonthlyInterval
+
+        def initialize(*)
+          @to     = Date.today.end_of_month
+          @from   = @to.years_ago(1).beginning_of_month
+          @format = '%d %B %Y'
+
+          super
+        end
+      end
+
+      class MonthChart < Chart
+        include DailyInterval
+
+        def initialize(*)
+          @to     = Date.today
+          @from   = @to - 30.days
+          @format = '%d %B'
+
+          super
+        end
+      end
+
+      class WeekChart < Chart
+        include DailyInterval
+
+        def initialize(*)
+          @to     = Date.today
+          @from   = @to - 7.days
+          @format = '%d %B'
+
+          super
+        end
+      end
+
+      class PipelineTime < Chart
+        def collect
+          commits = project.pipelines.last(30)
+
+          commits.each do |commit|
+            @labels << commit.short_sha
+            duration = commit.duration || 0
+            @pipeline_times << (duration / 60)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb
new file mode 100644
index 00000000000..0daddaa638c
--- /dev/null
+++ b/lib/gitlab/ci/mask_secret.rb
@@ -0,0 +1,12 @@
+module Gitlab
+  module Ci::MaskSecret
+    class << self
+      def mask!(value, token)
+        return value unless value.present? && token.present?
+
+        value.gsub!(token, 'x' * token.length)
+        value
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb
new file mode 100644
index 00000000000..3994a50772b
--- /dev/null
+++ b/lib/gitlab/ci/model.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Ci
+    module Model
+      def table_name_prefix
+        "ci_"
+      end
+
+      def model_name
+        @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index 8503ecf8700..ab3408f48d6 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -56,13 +56,13 @@ module Gitlab
         end
 
         def html_with_state(state = nil)
-          ::Ci::Ansi2html.convert(stream, state)
+          ::Gitlab::Ci::Ansi2html.convert(stream, state)
         end
 
         def html(last_lines: nil)
           text = raw(last_lines: last_lines)
           buffer = StringIO.new(text)
-          ::Ci::Ansi2html.convert(buffer).html
+          ::Gitlab::Ci::Ansi2html.convert(buffer).html
         end
 
         def extract_coverage(regex)
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
new file mode 100644
index 00000000000..7582964b24e
--- /dev/null
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -0,0 +1,253 @@
+module Gitlab
+  module Ci
+    class YamlProcessor
+      ValidationError = Class.new(StandardError)
+
+      include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
+
+      attr_reader :path, :cache, :stages, :jobs
+
+      def initialize(config, path = nil)
+        @ci_config = Gitlab::Ci::Config.new(config)
+        @config = @ci_config.to_hash
+        @path = path
+
+        unless @ci_config.valid?
+          raise ValidationError, @ci_config.errors.first
+        end
+
+        initial_parsing
+      rescue Gitlab::Ci::Config::Loader::FormatError => e
+        raise ValidationError, e.message
+      end
+
+      def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
+        jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
+          build_attributes(name)
+        end
+      end
+
+      def builds
+        @jobs.map do |name, _|
+          build_attributes(name)
+        end
+      end
+
+      def stage_seeds(pipeline)
+        seeds = @stages.uniq.map do |stage|
+          builds = pipeline_stage_builds(stage, pipeline)
+
+          Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
+        end
+
+        seeds.compact
+      end
+
+      def build_attributes(name)
+        job = @jobs[name.to_sym] || {}
+
+        { stage_idx: @stages.index(job[:stage]),
+          stage: job[:stage],
+          commands: job[:commands],
+          tag_list: job[:tags] || [],
+          name: job[:name].to_s,
+          allow_failure: job[:ignore],
+          when: job[:when] || 'on_success',
+          environment: job[:environment_name],
+          coverage_regex: job[:coverage],
+          yaml_variables: yaml_variables(name),
+          options: {
+            image: job[:image],
+            services: job[:services],
+            artifacts: job[:artifacts],
+            cache: job[:cache],
+            dependencies: job[:dependencies],
+            before_script: job[:before_script],
+            script: job[:script],
+            after_script: job[:after_script],
+            environment: job[:environment],
+            retry: job[:retry]
+          }.compact }
+      end
+
+      def self.validation_message(content)
+        return 'Please provide content of .gitlab-ci.yml' if content.blank?
+
+        begin
+          Gitlab::Ci::YamlProcessor.new(content)
+          nil
+        rescue ValidationError, Psych::SyntaxError => e
+          e.message
+        end
+      end
+
+      private
+
+      def pipeline_stage_builds(stage, pipeline)
+        builds = builds_for_stage_and_ref(
+          stage, pipeline.ref, pipeline.tag?, pipeline.source)
+
+        builds.select do |build|
+          job = @jobs[build.fetch(:name).to_sym]
+          has_kubernetes = pipeline.has_kubernetes_active?
+          only_kubernetes = job.dig(:only, :kubernetes)
+          except_kubernetes = job.dig(:except, :kubernetes)
+
+          [!only_kubernetes && !except_kubernetes,
+           only_kubernetes && has_kubernetes,
+           except_kubernetes && !has_kubernetes].any?
+        end
+      end
+
+      def jobs_for_ref(ref, tag = false, source = nil)
+        @jobs.select do |_, job|
+          process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source)
+        end
+      end
+
+      def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
+        jobs_for_ref(ref, tag, source).select do |_, job|
+          job[:stage] == stage
+        end
+      end
+
+      def initial_parsing
+        ##
+        # Global config
+        #
+        @before_script = @ci_config.before_script
+        @image = @ci_config.image
+        @after_script = @ci_config.after_script
+        @services = @ci_config.services
+        @variables = @ci_config.variables
+        @stages = @ci_config.stages
+        @cache = @ci_config.cache
+
+        ##
+        # Jobs
+        #
+        @jobs = @ci_config.jobs
+
+        @jobs.each do |name, job|
+          # logical validation for job
+
+          validate_job_stage!(name, job)
+          validate_job_dependencies!(name, job)
+          validate_job_environment!(name, job)
+        end
+      end
+
+      def yaml_variables(name)
+        variables = (@variables || {})
+          .merge(job_variables(name))
+
+        variables.map do |key, value|
+          { key: key.to_s, value: value, public: true }
+        end
+      end
+
+      def job_variables(name)
+        job = @jobs[name.to_sym]
+        return {} unless job
+
+        job[:variables] || {}
+      end
+
+      def validate_job_stage!(name, job)
+        return unless job[:stage]
+
+        unless job[:stage].is_a?(String) && job[:stage].in?(@stages)
+          raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}"
+        end
+      end
+
+      def validate_job_dependencies!(name, job)
+        return unless job[:dependencies]
+
+        stage_index = @stages.index(job[:stage])
+
+        job[:dependencies].each do |dependency|
+          raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym]
+
+          unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index
+            raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages"
+          end
+        end
+      end
+
+      def validate_job_environment!(name, job)
+        return unless job[:environment]
+        return unless job[:environment].is_a?(Hash)
+
+        environment = job[:environment]
+        validate_on_stop_job!(name, environment, environment[:on_stop])
+      end
+
+      def validate_on_stop_job!(name, environment, on_stop)
+        return unless on_stop
+
+        on_stop_job = @jobs[on_stop.to_sym]
+        unless on_stop_job
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined"
+        end
+
+        unless on_stop_job[:environment]
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined"
+        end
+
+        unless on_stop_job[:environment][:name] == environment[:name]
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name"
+        end
+
+        unless on_stop_job[:environment][:action] == 'stop'
+          raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined"
+        end
+      end
+
+      def process?(only_params, except_params, ref, tag, source)
+        if only_params.present?
+          return false unless matching?(only_params, ref, tag, source)
+        end
+
+        if except_params.present?
+          return false if matching?(except_params, ref, tag, source)
+        end
+
+        true
+      end
+
+      def matching?(patterns, ref, tag, source)
+        patterns.any? do |pattern|
+          pattern, path = pattern.split('@', 2)
+          matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
+        end
+      end
+
+      def matches_path?(path)
+        return true unless path
+
+        path == self.path
+      end
+
+      def matches_pattern?(pattern, ref, tag, source)
+        return true if tag && pattern == 'tags'
+        return true if !tag && pattern == 'branches'
+        return true if source_to_pattern(source) == pattern
+
+        if pattern.first == "/" && pattern.last == "/"
+          Regexp.new(pattern[1...-1]) =~ ref
+        else
+          pattern == ref
+        end
+      end
+
+      def source_to_pattern(source)
+        if %w[api external web].include?(source)
+          source
+        else
+          source&.pluralize
+        end
+      end
+    end
+  end
+end