Add latest changes from gitlab-org/gitlab@master

7 files changed, 115 insertions, 41 deletions

diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb
index 8d25b66af9c..908e560999e 100644
--- a/lib/gitlab/ci/ansi2json/converter.rb
+++ b/lib/gitlab/ci/ansi2json/converter.rb
@@ -22,11 +22,11 @@ module Gitlab
 
           start_offset = @state.offset
 
-          @state.set_current_line!(style: Style.new(@state.inherited_style))
+          @state.new_line!(
+            style: Style.new(@state.inherited_style))
 
           stream.each_line do |line|
-            s = StringScanner.new(line)
-            convert_line(s)
+            consume_line(line)
           end
 
           # This must be assigned before flushing the current line
@@ -52,26 +52,43 @@ module Gitlab
 
         private
 
-        def convert_line(scanner)
-          until scanner.eos?
-
-            if scanner.scan(Gitlab::Regex.build_trace_section_regex)
-              handle_section(scanner)
-            elsif scanner.scan(/\e([@-_])(.*?)([@-~])/)
-              handle_sequence(scanner)
-            elsif scanner.scan(/\e(([@-_])(.*?)?)?$/)
-              break
-            elsif scanner.scan(/</)
-              @state.current_line << '&lt;'
-            elsif scanner.scan(/\r?\n/)
-              # we advance the offset of the next current line
-              # so it does not start from \n
-              flush_current_line(advance_offset: scanner.matched_size)
-            else
-              @state.current_line << scanner.scan(/./m)
-            end
-
-            @state.offset += scanner.matched_size
+        def consume_line(line)
+          scanner = StringScanner.new(line)
+
+          consume_token(scanner) until scanner.eos?
+        end
+
+        def consume_token(scanner)
+          if scan_token(scanner, Gitlab::Regex.build_trace_section_regex, consume: false)
+            handle_section(scanner)
+          elsif scan_token(scanner, /\e([@-_])(.*?)([@-~])/)
+            handle_sequence(scanner)
+          elsif scan_token(scanner, /\e(([@-_])(.*?)?)?$/)
+            # stop scanning
+            scanner.terminate
+          elsif scan_token(scanner, /</)
+            @state.current_line << '&lt;'
+          elsif scan_token(scanner, /\r?\n/)
+            flush_current_line
+          elsif scan_token(scanner, /\r/)
+            # drop last line
+            @state.current_line.clear!
+          elsif scan_token(scanner, /.[^\e<\r\ns]*/m)
+            # this is a join from all previous tokens and first letters
+            # it always matches at least one character `.`
+            # it matches everything that is not start of:
+            # `\e`, `<`, `\r`, `\n`, `s` (for section_start)
+            @state.current_line << scanner[0]
+          else
+            raise 'invalid parser state'
+          end
+        end
+
+        def scan_token(scanner, match, consume: true)
+          scanner.scan(match).tap do |result|
+            # we need to move offset as soon
+            # as we match the token
+            @state.offset += scanner.matched_size if consume && result
           end
         end
 
@@ -96,32 +113,50 @@ module Gitlab
           section_name = sanitize_section_name(section)
 
           if action == "start"
-            handle_section_start(section_name, timestamp)
+            handle_section_start(scanner, section_name, timestamp)
           elsif action == "end"
-            handle_section_end(section_name, timestamp)
+            handle_section_end(scanner, section_name, timestamp)
+          else
+            raise 'unsupported action'
           end
         end
 
-        def handle_section_start(section, timestamp)
-          flush_current_line unless @state.current_line.empty?
+        def handle_section_start(scanner, section, timestamp)
+          # We make a new line for new section
+          flush_current_line
+
           @state.open_section(section, timestamp)
+
+          # we need to consume match after handling
+          # the open of section, as we want the section
+          # marker to be refresh on incremental update
+          @state.offset += scanner.matched_size
         end
 
-        def handle_section_end(section, timestamp)
+        def handle_section_end(scanner, section, timestamp)
           return unless @state.section_open?(section)
 
-          flush_current_line unless @state.current_line.empty?
+          # We flush the content to make the end
+          # of section to be a new line
+          flush_current_line
+
           @state.close_section(section, timestamp)
 
-          # ensure that section end is detached from the last
-          # line in the section
+          # we need to consume match before handling
+          # as we want the section close marker
+          # not to be refreshed on incremental update
+          @state.offset += scanner.matched_size
+
+          # this flushes an empty line with `section_duration`
           flush_current_line
         end
 
-        def flush_current_line(advance_offset: 0)
-          @lines << @state.current_line.to_h
+        def flush_current_line
+          unless @state.current_line.empty?
+            @lines << @state.current_line.to_h
+          end
 
-          @state.set_current_line!(advance_offset: advance_offset)
+          @state.new_line!
         end
 
         def sanitize_section_name(section)
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
index 173fb1df88e..21aa1f84353 100644
--- a/lib/gitlab/ci/ansi2json/line.rb
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -47,12 +47,17 @@ module Gitlab
           @current_segment.text << data
         end
 
+        def clear!
+          @segments.clear
+          @current_segment = Segment.new(style: style)
+        end
+
         def style
           @current_segment.style
         end
 
         def empty?
-          @segments.empty? && @current_segment.empty?
+          @segments.empty? && @current_segment.empty? && @section_duration.nil?
         end
 
         def update_style(ansi_commands)
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
index db7a9035b8b..7e1a8102a35 100644
--- a/lib/gitlab/ci/ansi2json/state.rb
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -46,9 +46,9 @@ module Gitlab
           @open_sections.key?(section)
         end
 
-        def set_current_line!(style: nil, advance_offset: 0)
+        def new_line!(style: nil)
           new_line = Line.new(
-            offset: @offset + advance_offset,
+            offset: @offset,
             style: style || @current_line.style,
             sections: @open_sections.keys
           )
diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb
index 02e368c1813..7a86fca3056 100644
--- a/lib/gitlab/ci/config/entry/commands.rb
+++ b/lib/gitlab/ci/config/entry/commands.rb
@@ -11,11 +11,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
 
           validations do
-            validates :config, array_of_strings_or_string: true
+            validates :config, string_or_nested_array_of_strings: true
           end
 
           def value
-            Array(@config)
+            Array(@config).flatten(1)
           end
         end
       end
diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb
index 9d25a82b521..285e18218b3 100644
--- a/lib/gitlab/ci/config/entry/script.rb
+++ b/lib/gitlab/ci/config/entry/script.rb
@@ -11,7 +11,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
 
           validations do
-            validates :config, array_of_strings: true
+            validates :config, nested_array_of_strings: true
+          end
+
+          def value
+            config.flatten(1)
           end
         end
       end
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index 8a04cca60d7..d1c23c41d35 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -228,6 +228,34 @@ module Gitlab
           end
         end
 
+        class NestedArrayOfStringsValidator < ArrayOfStringsOrStringValidator
+          def validate_each(record, attribute, value)
+            unless validate_nested_array_of_strings(value)
+              record.errors.add(attribute, 'should be an array containing strings and arrays of strings')
+            end
+          end
+
+          private
+
+          def validate_nested_array_of_strings(values)
+            values.is_a?(Array) && values.all? { |element| validate_array_of_strings_or_string(element) }
+          end
+        end
+
+        class StringOrNestedArrayOfStringsValidator < NestedArrayOfStringsValidator
+          def validate_each(record, attribute, value)
+            unless validate_string_or_nested_array_of_strings(value)
+              record.errors.add(attribute, 'should be a string or an array containing strings and arrays of strings')
+            end
+          end
+
+          private
+
+          def validate_string_or_nested_array_of_strings(values)
+            validate_string(values) || validate_nested_array_of_strings(values)
+          end
+        end
+
         class TypeValidator < ActiveModel::EachValidator
           def validate_each(record, attribute, value)
             type = options[:with]
diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb
index 9085835dee6..99029b54a69 100644
--- a/lib/google_api/cloud_platform/client.rb
+++ b/lib/google_api/cloud_platform/client.rb
@@ -12,6 +12,7 @@ module GoogleApi
       SCOPE = 'https://www.googleapis.com/auth/cloud-platform'
       LEAST_TOKEN_LIFE_TIME = 10.minutes
       CLUSTER_MASTER_AUTH_USERNAME = 'admin'
+      CLUSTER_IPV4_CIDR_BLOCK = '/16'
 
       class << self
         def session_key_for_token
@@ -97,7 +98,8 @@ module GoogleApi
               enabled: legacy_abac
             },
             ip_allocation_policy: {
-              use_ip_aliases: true
+              use_ip_aliases: true,
+              cluster_ipv4_cidr_block: CLUSTER_IPV4_CIDR_BLOCK
             },
             addons_config: enable_addons.each_with_object({}) do |addon, hash|
               hash[addon] = { disabled: false }