Add latest changes from gitlab-org/gitlab@master

5 files changed, 606 insertions, 0 deletions

diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb
new file mode 100644
index 00000000000..53adaf38b87
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/converter.rb
@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Converter
+        def convert(stream, new_state)
+          @lines = []
+          @state = State.new(new_state, stream.size)
+
+          append = false
+          truncated = false
+
+          cur_offset = stream.tell
+          if cur_offset > @state.offset
+            @state.offset = cur_offset
+            truncated = true
+          else
+            stream.seek(@state.offset)
+            append = @state.offset > 0
+          end
+
+          start_offset = @state.offset
+
+          @state.set_current_line!(style: Style.new(@state.inherited_style))
+
+          stream.each_line do |line|
+            s = StringScanner.new(line)
+            convert_line(s)
+          end
+
+          # This must be assigned before flushing the current line
+          # or the @current_line.offset will advance to the very end
+          # of the trace. Instead we want @last_line_offset to always
+          # point to the beginning of last line.
+          @state.set_last_line_offset
+
+          flush_current_line
+
+          OpenStruct.new(
+            lines: @lines,
+            state: @state.encode,
+            append: append,
+            truncated: truncated,
+            offset: start_offset,
+            size: stream.tell - start_offset,
+            total: stream.size
+          )
+        end
+
+        private
+
+        def convert_line(scanner)
+          until scanner.eos?
+
+            if scanner.scan(Gitlab::Regex.build_trace_section_regex)
+              handle_section(scanner)
+            elsif scanner.scan(/\e([@-_])(.*?)([@-~])/)
+              handle_sequence(scanner)
+            elsif scanner.scan(/\e(([@-_])(.*?)?)?$/)
+              break
+            elsif scanner.scan(/</)
+              @state.current_line << '&lt;'
+            elsif scanner.scan(/\r?\n/)
+              # we advance the offset of the next current line
+              # so it does not start from \n
+              flush_current_line(advance_offset: scanner.matched_size)
+            else
+              @state.current_line << scanner.scan(/./m)
+            end
+
+            @state.offset += scanner.matched_size
+          end
+        end
+
+        def handle_sequence(scanner)
+          indicator = scanner[1]
+          commands = scanner[2].split ';'
+          terminator = scanner[3]
+
+          # We are only interested in color and text style changes - triggered by
+          # sequences starting with '\e[' and ending with 'm'. Any other control
+          # sequence gets stripped (including stuff like "delete last line")
+          return unless indicator == '[' && terminator == 'm'
+
+          @state.update_style(commands)
+        end
+
+        def handle_section(scanner)
+          action = scanner[1]
+          timestamp = scanner[2]
+          section = scanner[3]
+
+          section_name = sanitize_section_name(section)
+
+          if action == "start"
+            handle_section_start(section_name, timestamp)
+          elsif action == "end"
+            handle_section_end(section_name, timestamp)
+          end
+        end
+
+        def handle_section_start(section, timestamp)
+          flush_current_line unless @state.current_line.empty?
+          @state.open_section(section, timestamp)
+        end
+
+        def handle_section_end(section, timestamp)
+          return unless @state.section_open?(section)
+
+          flush_current_line unless @state.current_line.empty?
+          @state.close_section(section, timestamp)
+
+          # ensure that section end is detached from the last
+          # line in the section
+          flush_current_line
+        end
+
+        def flush_current_line(advance_offset: 0)
+          @lines << @state.current_line.to_h
+
+          @state.set_current_line!(advance_offset: advance_offset)
+        end
+
+        def sanitize_section_name(section)
+          section.to_s.downcase.gsub(/[^a-z0-9]/, '-')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
new file mode 100644
index 00000000000..173fb1df88e
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      # Line class is responsible for keeping the internal state of
+      # a log line and to finally serialize it as Hash.
+      class Line
+        # Line::Segment is a portion of a line that has its own style
+        # and text. Multiple segments make the line content.
+        class Segment
+          attr_accessor :text, :style
+
+          def initialize(style:)
+            @text = +''
+            @style = style
+          end
+
+          def empty?
+            text.empty?
+          end
+
+          def to_h
+            # Without force encoding to UTF-8 we could get an error
+            # when serializing the Hash to JSON.
+            # Encoding::UndefinedConversionError:
+            #   "\xE2" from ASCII-8BIT to UTF-8
+            { text: text.force_encoding('UTF-8') }.tap do |result|
+              result[:style] = style.to_s if style.set?
+            end
+          end
+        end
+
+        attr_reader :offset, :sections, :segments, :current_segment,
+                    :section_header, :section_duration
+
+        def initialize(offset:, style:, sections: [])
+          @offset = offset
+          @segments = []
+          @sections = sections
+          @section_header = false
+          @duration = nil
+          @current_segment = Segment.new(style: style)
+        end
+
+        def <<(data)
+          @current_segment.text << data
+        end
+
+        def style
+          @current_segment.style
+        end
+
+        def empty?
+          @segments.empty? && @current_segment.empty?
+        end
+
+        def update_style(ansi_commands)
+          @current_segment.style.update(ansi_commands)
+        end
+
+        def add_section(section)
+          @sections << section
+        end
+
+        def set_as_section_header
+          @section_header = true
+        end
+
+        def set_section_duration(duration)
+          @section_duration = Time.at(duration.to_i).strftime('%M:%S')
+        end
+
+        def flush_current_segment!
+          return if @current_segment.empty?
+
+          @segments << @current_segment.to_h
+          @current_segment = Segment.new(style: @current_segment.style)
+        end
+
+        def to_h
+          flush_current_segment!
+
+          { offset: offset, content: @segments }.tap do |result|
+            result[:section] = sections.last if sections.any?
+            result[:section_header] = true if @section_header
+            result[:section_duration] = @section_duration if @section_duration
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb
new file mode 100644
index 00000000000..d428680fb2a
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/parser.rb
@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+# This Parser translates ANSI escape codes into human readable format.
+# It considers color and format changes.
+# Inspired by http://en.wikipedia.org/wiki/ANSI_escape_code
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Parser
+        # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
+        COLOR = {
+          0 => 'black', # not that this is gray in the intense color table
+          1 => 'red',
+          2 => 'green',
+          3 => 'yellow',
+          4 => 'blue',
+          5 => 'magenta',
+          6 => 'cyan',
+          7 => 'white' # not that this is gray in the dark (aka default) color table
+        }.freeze
+
+        STYLE_SWITCHES = {
+          bold:       0x01,
+          italic:     0x02,
+          underline:  0x04,
+          conceal:    0x08,
+          cross:      0x10
+        }.freeze
+
+        def self.bold?(mask)
+          mask & STYLE_SWITCHES[:bold] != 0
+        end
+
+        def self.matching_formats(mask)
+          formats = []
+          STYLE_SWITCHES.each do |text_format, flag|
+            formats << "term-#{text_format}" if mask & flag != 0
+          end
+
+          formats
+        end
+
+        def initialize(command, ansi_stack = nil)
+          @command = command
+          @ansi_stack = ansi_stack
+        end
+
+        def changes
+          if self.respond_to?("on_#{@command}")
+            send("on_#{@command}", @ansi_stack) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        # rubocop:disable Style/SingleLineMethods
+        def on_0(_) { reset: true } end
+
+        def on_1(_) { enable: STYLE_SWITCHES[:bold] } end
+
+        def on_3(_) { enable: STYLE_SWITCHES[:italic] } end
+
+        def on_4(_) { enable: STYLE_SWITCHES[:underline] } end
+
+        def on_8(_) { enable: STYLE_SWITCHES[:conceal] } end
+
+        def on_9(_) { enable: STYLE_SWITCHES[:cross] } end
+
+        def on_21(_) { disable: STYLE_SWITCHES[:bold] } end
+
+        def on_22(_) { disable: STYLE_SWITCHES[:bold] } end
+
+        def on_23(_) { disable: STYLE_SWITCHES[:italic] } end
+
+        def on_24(_) { disable: STYLE_SWITCHES[:underline] } end
+
+        def on_28(_) { disable: STYLE_SWITCHES[:conceal] } end
+
+        def on_29(_) { disable: STYLE_SWITCHES[:cross] } end
+
+        def on_30(_) { fg: fg_color(0) } end
+
+        def on_31(_) { fg: fg_color(1) } end
+
+        def on_32(_) { fg: fg_color(2) } end
+
+        def on_33(_) { fg: fg_color(3) } end
+
+        def on_34(_) { fg: fg_color(4) } end
+
+        def on_35(_) { fg: fg_color(5) } end
+
+        def on_36(_) { fg: fg_color(6) } end
+
+        def on_37(_) { fg: fg_color(7) } end
+
+        def on_38(stack) { fg: fg_color_256(stack) } end
+
+        def on_39(_) { fg: fg_color(9) } end
+
+        def on_40(_) { bg: bg_color(0) } end
+
+        def on_41(_) { bg: bg_color(1) } end
+
+        def on_42(_) { bg: bg_color(2) } end
+
+        def on_43(_) { bg: bg_color(3) } end
+
+        def on_44(_) { bg: bg_color(4) } end
+
+        def on_45(_) { bg: bg_color(5) } end
+
+        def on_46(_) { bg: bg_color(6) } end
+
+        def on_47(_) { bg: bg_color(7) } end
+
+        def on_48(stack) { bg: bg_color_256(stack) } end
+
+        # TODO: all the x9 never get called?
+        def on_49(_) { fg: fg_color(9) } end
+
+        def on_90(_) { fg: fg_color(0, 'l') } end
+
+        def on_91(_) { fg: fg_color(1, 'l') } end
+
+        def on_92(_) { fg: fg_color(2, 'l') } end
+
+        def on_93(_) { fg: fg_color(3, 'l') } end
+
+        def on_94(_) { fg: fg_color(4, 'l') } end
+
+        def on_95(_) { fg: fg_color(5, 'l') } end
+
+        def on_96(_) { fg: fg_color(6, 'l') } end
+
+        def on_97(_) { fg: fg_color(7, 'l') } end
+
+        def on_99(_) { fg: fg_color(9, 'l') } end
+
+        def on_100(_) { fg: bg_color(0, 'l') } end
+
+        def on_101(_) { fg: bg_color(1, 'l') } end
+
+        def on_102(_) { fg: bg_color(2, 'l') } end
+
+        def on_103(_) { fg: bg_color(3, 'l') } end
+
+        def on_104(_) { fg: bg_color(4, 'l') } end
+
+        def on_105(_) { fg: bg_color(5, 'l') } end
+
+        def on_106(_) { fg: bg_color(6, 'l') } end
+
+        def on_107(_) { fg: bg_color(7, 'l') } end
+
+        def on_109(_) { fg: bg_color(9, 'l') } end
+        # rubocop:enable Style/SingleLineMethods
+
+        def fg_color(color_index, prefix = nil)
+          term_color_class(color_index, ['fg', prefix])
+        end
+
+        def fg_color_256(command_stack)
+          xterm_color_class(command_stack, 'fg')
+        end
+
+        def bg_color(color_index, prefix = nil)
+          term_color_class(color_index, ['bg', prefix])
+        end
+
+        def bg_color_256(command_stack)
+          xterm_color_class(command_stack, 'bg')
+        end
+
+        def term_color_class(color_index, prefix)
+          color_name = COLOR[color_index]
+          return if color_name.nil?
+
+          color_class(['term', prefix, color_name])
+        end
+
+        def xterm_color_class(command_stack, prefix)
+          # the 38 and 48 commands have to be followed by "5" and the color index
+          return unless command_stack.length >= 2
+          return unless command_stack[0] == "5"
+
+          command_stack.shift # ignore the "5" command
+          color_index = command_stack.shift.to_i
+
+          return unless color_index >= 0
+          return unless color_index <= 255
+
+          color_class(["xterm", prefix, color_index])
+        end
+
+        def color_class(segments)
+          [segments].flatten.compact.join('-')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
new file mode 100644
index 00000000000..db7a9035b8b
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# In this class we keep track of the state changes that the
+# Converter makes as it scans through the log stream.
+module Gitlab
+  module Ci
+    module Ansi2json
+      class State
+        attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset
+
+        def initialize(new_state, stream_size)
+          @offset = 0
+          @inherited_style = {}
+          @open_sections = {}
+          @stream_size = stream_size
+
+          restore_state!(new_state)
+        end
+
+        def encode
+          state = {
+            offset: @last_line_offset,
+            style: @current_line.style.to_h,
+            open_sections: @open_sections
+          }
+          Base64.urlsafe_encode64(state.to_json)
+        end
+
+        def open_section(section, timestamp)
+          @open_sections[section] = timestamp
+
+          @current_line.add_section(section)
+          @current_line.set_as_section_header
+        end
+
+        def close_section(section, timestamp)
+          return unless section_open?(section)
+
+          duration = timestamp.to_i - @open_sections[section].to_i
+          @current_line.set_section_duration(duration)
+
+          @open_sections.delete(section)
+        end
+
+        def section_open?(section)
+          @open_sections.key?(section)
+        end
+
+        def set_current_line!(style: nil, advance_offset: 0)
+          new_line = Line.new(
+            offset: @offset + advance_offset,
+            style: style || @current_line.style,
+            sections: @open_sections.keys
+          )
+          @current_line = new_line
+        end
+
+        def set_last_line_offset
+          @last_line_offset = @current_line.offset
+        end
+
+        def update_style(commands)
+          @current_line.flush_current_segment!
+          @current_line.update_style(commands)
+        end
+
+        private
+
+        def restore_state!(encoded_state)
+          state = decode_state(encoded_state)
+
+          return unless state
+          return if state['offset'].to_i > @stream_size
+
+          @offset = state['offset'].to_i if state['offset']
+          @open_sections = state['open_sections'] if state['open_sections']
+
+          if state['style']
+            @inherited_style = {
+              fg: state.dig('style', 'fg'),
+              bg: state.dig('style', 'bg'),
+              mask: state.dig('style', 'mask')
+            }
+          end
+        end
+
+        def decode_state(state)
+          return unless state.present?
+
+          decoded_state = Base64.urlsafe_decode64(state)
+          return unless decoded_state.present?
+
+          JSON.parse(decoded_state)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/ansi2json/style.rb b/lib/gitlab/ci/ansi2json/style.rb
new file mode 100644
index 00000000000..2739ffdfa5d
--- /dev/null
+++ b/lib/gitlab/ci/ansi2json/style.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Ansi2json
+      class Style
+        attr_reader :fg, :bg, :mask
+
+        def initialize(fg: nil, bg: nil, mask: 0)
+          @fg = fg
+          @bg = bg
+          @mask = mask
+
+          update_formats
+        end
+
+        def update(ansi_commands)
+          command = ansi_commands.shift
+          return unless command
+
+          if changes = Gitlab::Ci::Ansi2json::Parser.new(command, ansi_commands).changes
+            apply_changes(changes)
+          end
+
+          update(ansi_commands)
+        end
+
+        def set?
+          @fg || @bg || @formats.any?
+        end
+
+        def reset!
+          @fg = nil
+          @bg = nil
+          @mask = 0
+          @formats = []
+        end
+
+        def ==(other)
+          self.to_h == other.to_h
+        end
+
+        def to_s
+          [@fg, @bg, @formats].flatten.compact.join(' ')
+        end
+
+        def to_h
+          { fg: @fg, bg: @bg, mask: @mask }
+        end
+
+        private
+
+        def apply_changes(changes)
+          case
+          when changes[:reset]
+            reset!
+          when changes[:fg]
+            @fg = changes[:fg]
+          when changes[:bg]
+            @bg = changes[:bg]
+          when changes[:enable]
+            @mask |= changes[:enable]
+          when changes[:disable]
+            @mask &= ~changes[:disable]
+          else
+            return
+          end
+
+          update_formats
+        end
+
+        def update_formats
+          # Most terminals show bold colored text in the light color variant
+          # Let's mimic that here
+          if @fg.present? && Gitlab::Ci::Ansi2json::Parser.bold?(@mask)
+            @fg = @fg.sub(/fg-([a-z]{2,}+)/, 'fg-l-\1')
+          end
+
+          @formats = Gitlab::Ci::Ansi2json::Parser.matching_formats(@mask)
+        end
+      end
+    end
+  end
+end