Merge remote-tracking branch 'upstream/master' into artifacts-from-ref-and-build-name

* upstream/master: (359 commits)
  Add new image to show the 'Reset template' button
  Refactor description templates documentation
  Remove index from pipeline toggles
  Hide `Create new list button` on Issues and MRs pages
  Remove params from build; general refactor
  Style build container box; add check mark to active build
  Display jobs as scrolling list in sidebar
  Move stages and jobs to build sidebar
  Removed vue assets
  Move skipped tests to end of array
  Remove unused data attributes
  Update changelog
  Style build dropdowns
  Populate dropdowns with current build on pageload
  Selecting stage updates builds dropdown
  Add data attributes to builds
  Change active state of list items; style dropdown items
  Hide dropdown if all tests fit on one line; add counter to dropdown
  Add overflow tests to dropdown
  Order by build status
  ...

11 files changed, 390 insertions, 9 deletions

diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 260ac81f5fa..9f3b582a263 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -20,8 +20,13 @@ module Ci
           build = Ci::RegisterBuildService.new.execute(current_runner)
 
           if build
+            Gitlab::Metrics.add_event(:build_found,
+                                      project: build.project.path_with_namespace)
+
             present build, with: Entities::BuildDetails
           else
+            Gitlab::Metrics.add_event(:build_not_found)
+
             not_found!
           end
         end
@@ -42,6 +47,9 @@ module Ci
 
           build.update_attributes(trace: params[:trace]) if params[:trace]
 
+          Gitlab::Metrics.add_event(:update_build,
+                                    project: build.project.path_with_namespace)
+
           case params[:state].to_s
           when 'success'
             build.success
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
new file mode 100644
index 00000000000..0a1fd27ced5
--- /dev/null
+++ b/lib/gitlab/conflict/file.rb
@@ -0,0 +1,186 @@
+module Gitlab
+  module Conflict
+    class File
+      include Gitlab::Routing.url_helpers
+      include IconsHelper
+
+      class MissingResolution < StandardError
+      end
+
+      CONTEXT_LINES = 3
+
+      attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository
+
+      def initialize(merge_file_result, conflict, merge_request:)
+        @merge_file_result = merge_file_result
+        @their_path = conflict[:theirs][:path]
+        @our_path = conflict[:ours][:path]
+        @our_mode = conflict[:ours][:mode]
+        @merge_request = merge_request
+        @repository = merge_request.project.repository
+        @match_line_headers = {}
+      end
+
+      # Array of Gitlab::Diff::Line objects
+      def lines
+        @lines ||= Gitlab::Conflict::Parser.new.parse(merge_file_result[:data],
+                                                      our_path: our_path,
+                                                      their_path: their_path,
+                                                      parent_file: self)
+      end
+
+      def resolve_lines(resolution)
+        section_id = nil
+
+        lines.map do |line|
+          unless line.type
+            section_id = nil
+            next line
+          end
+
+          section_id ||= line_code(line)
+
+          case resolution[section_id]
+          when 'head'
+            next unless line.type == 'new'
+          when 'origin'
+            next unless line.type == 'old'
+          else
+            raise MissingResolution, "Missing resolution for section ID: #{section_id}"
+          end
+
+          line
+        end.compact
+      end
+
+      def highlight_lines!
+        their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n")
+        our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n")
+
+        their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines
+        our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines
+
+        lines.each do |line|
+          if line.type == 'old'
+            line.rich_text = their_highlight[line.old_line - 1].try(:html_safe)
+          else
+            line.rich_text = our_highlight[line.new_line - 1].try(:html_safe)
+          end
+        end
+      end
+
+      def sections
+        return @sections if @sections
+
+        chunked_lines = lines.chunk { |line| line.type.nil? }.to_a
+        match_line = nil
+
+        sections_count = chunked_lines.size
+
+        @sections = chunked_lines.flat_map.with_index do |(no_conflict, lines), i|
+          section = nil
+
+          # We need to reduce context sections to CONTEXT_LINES. Conflict sections are
+          # always shown in full.
+          if no_conflict
+            conflict_before = i > 0
+            conflict_after = (sections_count - i) > 1
+
+            if conflict_before && conflict_after
+              # Create a gap in a long context section.
+              if lines.length > CONTEXT_LINES * 2
+                head_lines = lines.first(CONTEXT_LINES)
+                tail_lines = lines.last(CONTEXT_LINES)
+
+                # Ensure any existing match line has text for all lines up to the last
+                # line of its context.
+                update_match_line_text(match_line, head_lines.last)
+
+                # Insert a new match line after the created gap.
+                match_line = create_match_line(tail_lines.first)
+
+                section = [
+                  { conflict: false, lines: head_lines },
+                  { conflict: false, lines: tail_lines.unshift(match_line) }
+                ]
+              end
+            elsif conflict_after
+              tail_lines = lines.last(CONTEXT_LINES)
+
+              # Create a gap and insert a match line at the start.
+              if lines.length > tail_lines.length
+                match_line = create_match_line(tail_lines.first)
+
+                tail_lines.unshift(match_line)
+              end
+
+              lines = tail_lines
+            elsif conflict_before
+              # We're at the end of the file (no conflicts after), so just remove extra
+              # trailing lines.
+              lines = lines.first(CONTEXT_LINES)
+            end
+          end
+
+          # We want to update the match line's text every time unless we've already
+          # created a gap and its corresponding match line.
+          update_match_line_text(match_line, lines.last) unless section
+
+          section ||= { conflict: !no_conflict, lines: lines }
+          section[:id] = line_code(lines.first) unless no_conflict
+          section
+        end
+      end
+
+      def line_code(line)
+        Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos)
+      end
+
+      def create_match_line(line)
+        Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos)
+      end
+
+      # Any line beginning with a letter, an underscore, or a dollar can be used in a
+      # match line header. Only context sections can contain match lines, as match lines
+      # have to exist in both versions of the file.
+      def find_match_line_header(index)
+        return @match_line_headers[index] if @match_line_headers.key?(index)
+
+        @match_line_headers[index] = begin
+          if index >= 0
+            line = lines[index]
+
+            if line.type.nil? && line.text.match(/\A[A-Za-z$_]/)
+              " #{line.text}"
+            else
+              find_match_line_header(index - 1)
+            end
+          end
+        end
+      end
+
+      # Set the match line's text for the current line. A match line takes its start
+      # position and context header (where present) from itself, and its end position from
+      # the line passed in.
+      def update_match_line_text(match_line, line)
+        return unless match_line
+
+        header = find_match_line_header(match_line.index - 1)
+
+        match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}"
+      end
+
+      def as_json(opts = nil)
+        {
+          old_path: their_path,
+          new_path: our_path,
+          blob_icon: file_type_icon_class('file', our_mode, our_path),
+          blob_path: namespace_project_blob_path(merge_request.project.namespace,
+                                                 merge_request.project,
+                                                 ::File.join(merge_request.diff_refs.head_sha, our_path)),
+          sections: sections
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
new file mode 100644
index 00000000000..bbd0427a2c8
--- /dev/null
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -0,0 +1,57 @@
+module Gitlab
+  module Conflict
+    class FileCollection
+      class ConflictSideMissing < StandardError
+      end
+
+      attr_reader :merge_request, :our_commit, :their_commit
+
+      def initialize(merge_request)
+        @merge_request = merge_request
+        @our_commit = merge_request.source_branch_head.raw.raw_commit
+        @their_commit = merge_request.target_branch_head.raw.raw_commit
+      end
+
+      def repository
+        merge_request.project.repository
+      end
+
+      def merge_index
+        @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit)
+      end
+
+      def files
+        @files ||= merge_index.conflicts.map do |conflict|
+          raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours]
+
+          Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]),
+                                     conflict,
+                                     merge_request: merge_request)
+        end
+      end
+
+      def as_json(opts = nil)
+        {
+          target_branch: merge_request.target_branch,
+          source_branch: merge_request.source_branch,
+          commit_sha: merge_request.diff_head_sha,
+          commit_message: default_commit_message,
+          files: files
+        }
+      end
+
+      def default_commit_message
+        conflict_filenames = merge_index.conflicts.map do |conflict|
+          "#   #{conflict[:ours][:path]}"
+        end
+
+        <<EOM.chomp
+Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branch}'
+
+# Conflicts:
+#{conflict_filenames.join("\n")}
+EOM
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb
new file mode 100644
index 00000000000..6eccded7872
--- /dev/null
+++ b/lib/gitlab/conflict/parser.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module Conflict
+    class Parser
+      class ParserError < StandardError
+      end
+
+      class UnexpectedDelimiter < ParserError
+      end
+
+      class MissingEndDelimiter < ParserError
+      end
+
+      class UnmergeableFile < ParserError
+      end
+
+      def parse(text, our_path:, their_path:, parent_file: nil)
+        raise UnmergeableFile if text.blank? # Typically a binary file
+        raise UnmergeableFile if text.length > 102400
+
+        line_obj_index = 0
+        line_old = 1
+        line_new = 1
+        type = nil
+        lines = []
+        conflict_start = "<<<<<<< #{our_path}"
+        conflict_middle = '======='
+        conflict_end = ">>>>>>> #{their_path}"
+
+        text.each_line.map do |line|
+          full_line = line.delete("\n")
+
+          if full_line == conflict_start
+            raise UnexpectedDelimiter unless type.nil?
+
+            type = 'new'
+          elsif full_line == conflict_middle
+            raise UnexpectedDelimiter unless type == 'new'
+
+            type = 'old'
+          elsif full_line == conflict_end
+            raise UnexpectedDelimiter unless type == 'old'
+
+            type = nil
+          elsif line[0] == '\\'
+            type = 'nonewline'
+            lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file)
+          else
+            lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file)
+            line_old += 1 if type != 'new'
+            line_new += 1 if type != 'old'
+
+            line_obj_index += 1
+          end
+        end
+
+        raise MissingEndDelimiter unless type.nil?
+
+        lines
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index cf097e0d0de..80a146b4a5a 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -2,11 +2,13 @@ module Gitlab
   module Diff
     class Line
       attr_reader :type, :index, :old_pos, :new_pos
+      attr_writer :rich_text
       attr_accessor :text
 
-      def initialize(text, type, index, old_pos, new_pos)
+      def initialize(text, type, index, old_pos, new_pos, parent_file: nil)
         @text, @type, @index = text, type, index
         @old_pos, @new_pos = old_pos, new_pos
+        @parent_file = parent_file
       end
 
       def self.init_from_hash(hash)
@@ -43,9 +45,25 @@ module Gitlab
         type == 'old'
       end
 
+      def rich_text
+        @parent_file.highlight_lines! if @parent_file && !@rich_text
+
+        @rich_text
+      end
+
       def meta?
         type == 'match' || type == 'nonewline'
       end
+
+      def as_json(opts = nil)
+        {
+          type: type,
+          old_line: old_line,
+          new_line: new_line,
+          text: text,
+          rich_text: rich_text || text
+        }
+      end
     end
   end
 end
diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb
index 4446e921e0d..40a4815a9a0 100644
--- a/lib/gitlab/downtime_check/message.rb
+++ b/lib/gitlab/downtime_check/message.rb
@@ -1,10 +1,10 @@
 module Gitlab
   class DowntimeCheck
     class Message
-      attr_reader :path, :offline, :reason
+      attr_reader :path, :offline
 
-      OFFLINE = "\e[32moffline\e[0m"
-      ONLINE = "\e[31monline\e[0m"
+      OFFLINE = "\e[31moffline\e[0m"
+      ONLINE = "\e[32monline\e[0m"
 
       # path - The file path of the migration.
       # offline - When set to `true` the migration will require downtime.
@@ -19,10 +19,21 @@ module Gitlab
         label = offline ? OFFLINE : ONLINE
 
         message = "[#{label}]: #{path}"
-        message += ": #{reason}" if reason
+
+        if reason?
+          message += ":\n\n#{reason}\n\n"
+        end
 
         message
       end
+
+      def reason?
+        @reason.present?
+      end
+
+      def reason
+        @reason.strip.lines.map(&:strip).join("\n")
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 41fcd971c22..3d1ba33ec68 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -124,6 +124,15 @@ module Gitlab
       trans.action = action if trans
     end
 
+    # Tracks an event.
+    #
+    # See `Gitlab::Metrics::Transaction#add_event` for more details.
+    def self.add_event(*args)
+      trans = current_transaction
+
+      trans.add_event(*args) if trans
+    end
+
     # Returns the prefix to use for the name of a series.
     def self.series_prefix
       @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index f23d67e1e38..bd0afe53c51 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -4,15 +4,20 @@ module Gitlab
     class Metric
       JITTER_RANGE = 0.000001..0.001
 
-      attr_reader :series, :values, :tags
+      attr_reader :series, :values, :tags, :type
 
       # series - The name of the series (as a String) to store the metric in.
       # values - A Hash containing the values to store.
       # tags   - A Hash containing extra tags to add to the metrics.
-      def initialize(series, values, tags = {})
+      def initialize(series, values, tags = {}, type = :metric)
         @values = values
         @series = series
         @tags   = tags
+        @type   = type
+      end
+
+      def event?
+        type == :event
       end
 
       # Returns a Hash in a format that can be directly written to InfluxDB.
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index e61670f491c..b4493bf44d2 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -17,6 +17,10 @@ module Gitlab
         begin
           retval = trans.run { @app.call(env) }
 
+        rescue Exception => error # rubocop: disable Lint/RescueException
+          trans.add_event(:rails_exception)
+
+          raise error
         # Even in the event of an error we want to submit any metrics we
         # might've gathered up to this point.
         ensure
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index a1240fd33ee..f9dd8e41912 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -11,6 +11,10 @@ module Gitlab
           # Old gitlad-shell messages don't provide enqueued_at/created_at attributes
           trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
           trans.run { yield }
+        rescue Exception => error # rubocop: disable Lint/RescueException
+          trans.add_event(:sidekiq_exception)
+
+          raise error
         ensure
           trans.finish
         end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 968f3218950..7bc16181be6 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -4,7 +4,10 @@ module Gitlab
     class Transaction
       THREAD_KEY = :_gitlab_metrics_transaction
 
-      attr_reader :tags, :values, :methods
+      # The series to store events (e.g. Git pushes) in.
+      EVENT_SERIES = 'events'
+
+      attr_reader :tags, :values, :method, :metrics
 
       attr_accessor :action
 
@@ -55,6 +58,20 @@ module Gitlab
         @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags)
       end
 
+      # Tracks a business level event
+      #
+      # Business level events including events such as Git pushes, Emails being
+      # sent, etc.
+      #
+      # event_name - The name of the event (e.g. "git_push").
+      # tags - A set of tags to attach to the event.
+      def add_event(event_name, tags = {})
+        @metrics << Metric.new(EVENT_SERIES,
+                               { count: 1 },
+                               { event: event_name }.merge(tags),
+                               :event)
+      end
+
       # Returns a MethodCall object for the given name.
       def method_call_for(name)
         unless method = @methods[name]
@@ -101,7 +118,7 @@ module Gitlab
         submit_hashes = submit.map do |metric|
           hash = metric.to_hash
 
-          hash[:tags][:action] ||= @action if @action
+          hash[:tags][:action] ||= @action if @action && !metric.event?
 
           hash
         end