Merge remote-tracking branch 'upstream/master' into 27377-preload-pipeline-entity27377-preload-pipeline-entity

* upstream/master: (2534 commits)
  Update VERSION to 9.3.0-pre
  Update CHANGELOG.md for 9.2.0
  removes unnecessary redundacy in usage ping doc
  Respect the typo as rubocop said
  Add a test to ensure this works on MySQL
  Change pipelines schedules help page path
  change domain to hostname in usage ping doc
  Fixes broken MySQL migration for retried
  Show password field mask while editing service settings
  Add notes for supported schedulers and cloud providers
  Move environment monitoring to environments doc
  Add docs for change of Cache/Artifact restore order"
  Avoid resource intensive login checks if password is not provided
  Change translation for 'coding' by 'desarrollo' for Spanish
  Add to docs: issues multiple assignees
  rename "Add emoji" and "Award emoji" to "Add reaction" where appropriate
  Add project and group notification settings info
  32570 Fix border-bottom for project activity tab
  Add users endpoint to frontend API class
  Rename users on mysql
  ...

4 files changed, 58 insertions, 41 deletions

diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
index b3ccad7b28d..55402101e43 100644
--- a/lib/ci/ansi2html.rb
+++ b/lib/ci/ansi2html.rb
@@ -20,7 +20,7 @@ module Ci
       italic:     0x02,
       underline:  0x04,
       conceal:    0x08,
-      cross:      0x10,
+      cross:      0x10
     }.freeze
 
     def self.convert(ansi, state = nil)
@@ -132,34 +132,54 @@ module Ci
 
       STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
 
-      def convert(raw, new_state)
+      def convert(stream, new_state)
         reset_state
-        restore_state(raw, new_state) if new_state.present?
-
-        start = @offset
-        ansi = raw[@offset..-1]
+        restore_state(new_state, stream) if new_state.present?
+
+        append = false
+        truncated = false
+
+        cur_offset = stream.tell
+        if cur_offset > @offset
+          @offset = cur_offset
+          truncated = true
+        else
+          stream.seek(@offset)
+          append = @offset > 0
+        end
+        start_offset = @offset
 
         open_new_tag
 
-        s = StringScanner.new(ansi)
-        until s.eos?
-          if s.scan(/\e([@-_])(.*?)([@-~])/)
-            handle_sequence(s)
-          elsif s.scan(/\e(([@-_])(.*?)?)?$/)
-            break
-          elsif s.scan(/</)
-            @out << '&lt;'
-          elsif s.scan(/\r?\n/)
-            @out << '<br>'
-          else
-            @out << s.scan(/./m)
+        stream.each_line do |line|
+          s = StringScanner.new(line)
+          until s.eos?
+            if s.scan(/\e([@-_])(.*?)([@-~])/)
+              handle_sequence(s)
+            elsif s.scan(/\e(([@-_])(.*?)?)?$/)
+              break
+            elsif s.scan(/</)
+              @out << '&lt;'
+            elsif s.scan(/\r?\n/)
+              @out << '<br>'
+            else
+              @out << s.scan(/./m)
+            end
+            @offset += s.matched_size
           end
-          @offset += s.matched_size
         end
 
         close_open_tags()
 
-        { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 }
+        OpenStruct.new(
+          html: @out.force_encoding(Encoding.default_external),
+          state: state,
+          append: append,
+          truncated: truncated,
+          offset: start_offset,
+          size: stream.tell - start_offset,
+          total: stream.size
+        )
       end
 
       def handle_sequence(s)
@@ -240,10 +260,10 @@ module Ci
         Base64.urlsafe_encode64(state.to_json)
       end
 
-      def restore_state(raw, new_state)
+      def restore_state(new_state, stream)
         state = Base64.urlsafe_decode64(new_state)
         state = JSON.parse(state, symbolize_names: true)
-        return if state[:offset].to_i > raw.length
+        return if state[:offset].to_i > stream.size
 
         STATE_PARAMS.each do |param|
           send("#{param}=".to_sym, state[param])
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 746e76a1b1f..67b269b330c 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -61,7 +61,7 @@ module Ci
 
           update_runner_info
 
-          build.update_attributes(trace: params[:trace]) if params[:trace]
+          build.trace.set(params[:trace]) if params[:trace]
 
           Gitlab::Metrics.add_event(:update_build,
                                     project: build.project.path_with_namespace)
@@ -86,23 +86,20 @@ module Ci
         # Example Request:
         #   PATCH /builds/:id/trace.txt
         patch ":id/trace.txt" do
-          build = Ci::Build.find_by_id(params[:id])
-          authenticate_build!(build)
+          build = authenticate_build!
 
           error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
           content_range = request.headers['Content-Range']
           content_range = content_range.split('-')
 
-          current_length = build.trace_length
-          unless current_length == content_range[0].to_i
-            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
+          stream_size = build.trace.append(request.body.read, content_range[0].to_i)
+          if stream_size < 0
+            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
           end
 
-          build.append_trace(request.body.read, content_range[0].to_i)
-
           status 202
           header 'Build-Status', build.status
-          header 'Range', "0-#{build.trace_length}"
+          header 'Range', "0-#{stream_size}"
         end
 
         # Authorize artifacts uploading for build - Runners only
@@ -117,8 +114,7 @@ module Ci
           require_gitlab_workhorse!
           Gitlab::Workhorse.verify_api_request!(headers)
           not_allowed! unless Gitlab.config.artifacts.enabled
-          build = Ci::Build.find_by_id(params[:id])
-          authenticate_build!(build)
+          build = authenticate_build!
           forbidden!('build is not running') unless build.running?
 
           if params[:filesize]
@@ -154,8 +150,7 @@ module Ci
         post ":id/artifacts" do
           require_gitlab_workhorse!
           not_allowed! unless Gitlab.config.artifacts.enabled
-          build = Ci::Build.find_by_id(params[:id])
-          authenticate_build!(build)
+          build = authenticate_build!
           forbidden!('Build is not running!') unless build.running?
 
           artifacts_upload_path = ArtifactUploader.artifacts_upload_path
@@ -189,8 +184,7 @@ module Ci
         # Example Request:
         #   GET /builds/:id/artifacts
         get ":id/artifacts" do
-          build = Ci::Build.find_by_id(params[:id])
-          authenticate_build!(build)
+          build = authenticate_build!
           artifacts_file = build.artifacts_file
 
           unless artifacts_file.file_storage?
@@ -214,8 +208,7 @@ module Ci
         # Example Request:
         #   DELETE /builds/:id/artifacts
         delete ":id/artifacts" do
-          build = Ci::Build.find_by_id(params[:id])
-          authenticate_build!(build)
+          build = authenticate_build!
 
           status(200)
           build.erase_artifacts!
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 996990b464f..5109dc9670f 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -13,10 +13,14 @@ module Ci
         forbidden! unless current_runner
       end
 
-      def authenticate_build!(build)
+      def authenticate_build!
+        build = Ci::Build.find_by_id(params[:id])
+
         validate_build!(build) do
           forbidden! unless build_token_valid?(build)
         end
+
+        build
       end
 
       def validate_build!(build)
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 15a461a16dd..b06474cda7f 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -70,7 +70,7 @@ module Ci
           cache: job[:cache],
           dependencies: job[:dependencies],
           after_script: job[:after_script],
-          environment: job[:environment],
+          environment: job[:environment]
         }.compact
       }
     end