Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into zj/gitlab-ce-zj-auto-devops-table

28 files changed, 548 insertions, 266 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 94df543853b..1405a5d0f0e 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -108,6 +108,7 @@ module API
     mount ::API::Internal
     mount ::API::Issues
     mount ::API::Jobs
+    mount ::API::JobArtifacts
     mount ::API::Keys
     mount ::API::Labels
     mount ::API::Lint
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index ea78737288a..4b8d248f5f7 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -104,7 +104,7 @@ module API
 
         not_found! 'Commit' unless commit
 
-        commit.raw_diffs.to_a
+        present commit.raw_diffs.to_a, with: Entities::RepoDiff
       end
 
       desc "Get a commit's comments" do
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 9114b69606b..1d224d7bc21 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -291,10 +291,11 @@ module API
     end
 
     class RepoDiff < Grape::Entity
-      expose :old_path, :new_path, :a_mode, :b_mode, :diff
+      expose :old_path, :new_path, :a_mode, :b_mode
       expose :new_file?, as: :new_file
       expose :renamed_file?, as: :renamed_file
       expose :deleted_file?, as: :deleted_file
+      expose :json_safe_diff, as: :diff
     end
 
     class ProtectedRefAccess < Grape::Entity
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 3d377fdb9eb..e646c63467a 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -128,6 +128,10 @@ module API
       merge_request
     end
 
+    def find_build!(id)
+      user_project.builds.find(id.to_i)
+    end
+
     def authenticate!
       unauthorized! unless current_user && can?(initial_current_user, :access_api)
     end
@@ -160,6 +164,14 @@ module API
       authorize! :admin_project, user_project
     end
 
+    def authorize_read_builds!
+      authorize! :read_build, user_project
+    end
+
+    def authorize_update_builds!
+      authorize! :update_build, user_project
+    end
+
     def require_gitlab_workhorse!
       unless env['HTTP_GITLAB_WORKHORSE'].present?
         forbidden!('Request should be executed via GitLab Workhorse')
@@ -210,7 +222,7 @@ module API
 
     def bad_request!(attribute)
       message = ["400 (Bad request)"]
-      message << "\"" + attribute.to_s + "\" not given"
+      message << "\"" + attribute.to_s + "\" not given" if attribute
       render_api_error!(message.join(' '), 400)
     end
 
@@ -432,6 +444,10 @@ module API
       header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format))
     end
 
+    def send_artifacts_entry(build, entry)
+      header(*Gitlab::Workhorse.send_artifacts_entry(build, entry))
+    end
+
     # The Grape Error Middleware only has access to env but no params. We workaround this by
     # defining a method that returns the right value.
     def define_params_for_grape_middleware
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index f57ff0f2632..4c0db4d42b1 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -46,6 +46,15 @@ module API
         ::MergeRequests::GetUrlsService.new(project).execute(params[:changes])
       end
 
+      def redis_ping
+        result = Gitlab::Redis::SharedState.with { |redis| redis.ping }
+
+        result == 'PONG'
+      rescue => e
+        Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}")
+        false
+      end
+
       private
 
       def set_project
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 622bd9650e4..c0fef56378f 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -88,7 +88,8 @@ module API
         {
           api_version: API.version,
           gitlab_version: Gitlab::VERSION,
-          gitlab_rev: Gitlab::REVISION
+          gitlab_rev: Gitlab::REVISION,
+          redis: redis_ping
         }
       end
 
@@ -142,6 +143,14 @@ module API
         { success: true, recovery_codes: codes }
       end
 
+      post '/pre_receive' do
+        status 200
+
+        reference_counter_increased = Gitlab::ReferenceCounter.new(params[:gl_repository]).increase
+
+        { reference_counter_increased: reference_counter_increased }
+      end
+
       post "/notify_post_receive" do
         status 200
 
diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb
new file mode 100644
index 00000000000..2a8fa7659bf
--- /dev/null
+++ b/lib/api/job_artifacts.rb
@@ -0,0 +1,80 @@
+module API
+  class JobArtifacts < Grape::API
+    before { authenticate_non_get! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      desc 'Download the artifacts file from a job' do
+        detail 'This feature was introduced in GitLab 8.10'
+      end
+      params do
+        requires :ref_name, type: String, desc: 'The ref from repository'
+        requires :job,      type: String, desc: 'The name for the job'
+      end
+      get ':id/jobs/artifacts/:ref_name/download',
+        requirements: { ref_name: /.+/ } do
+        authorize_read_builds!
+
+        builds = user_project.latest_successful_builds_for(params[:ref_name])
+        latest_build = builds.find_by!(name: params[:job])
+
+        present_artifacts!(latest_build.artifacts_file)
+      end
+
+      desc 'Download the artifacts file from a job' do
+        detail 'This feature was introduced in GitLab 8.5'
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      get ':id/jobs/:job_id/artifacts' do
+        authorize_read_builds!
+
+        build = find_build!(params[:job_id])
+
+        present_artifacts!(build.artifacts_file)
+      end
+
+      desc 'Download a specific file from artifacts archive' do
+        detail 'This feature was introduced in GitLab 10.0'
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+        requires :artifact_path, type: String, desc: 'Artifact path'
+      end
+      get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do
+        authorize_read_builds!
+
+        build = find_build!(params[:job_id])
+        not_found! unless build.artifacts?
+
+        path = Gitlab::Ci::Build::Artifacts::Path
+          .new(params[:artifact_path])
+        bad_request! unless path.valid?
+
+        send_artifacts_entry(build, path)
+      end
+
+      desc 'Keep the artifacts to prevent them from being deleted' do
+        success Entities::Job
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      post ':id/jobs/:job_id/artifacts/keep' do
+        authorize_update_builds!
+
+        build = find_build!(params[:job_id])
+        authorize!(:update_build, build)
+        return not_found!(build) unless build.artifacts?
+
+        build.keep_artifacts!
+
+        status 200
+        present build, with: Entities::Job
+      end
+    end
+  end
+end
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index 5bab96398fd..3c1c412ba42 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -66,42 +66,11 @@ module API
       get ':id/jobs/:job_id' do
         authorize_read_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
 
         present build, with: Entities::Job
       end
 
-      desc 'Download the artifacts file from a job' do
-        detail 'This feature was introduced in GitLab 8.5'
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      get ':id/jobs/:job_id/artifacts' do
-        authorize_read_builds!
-
-        build = get_build!(params[:job_id])
-
-        present_artifacts!(build.artifacts_file)
-      end
-
-      desc 'Download the artifacts file from a job' do
-        detail 'This feature was introduced in GitLab 8.10'
-      end
-      params do
-        requires :ref_name, type: String, desc: 'The ref from repository'
-        requires :job,      type: String, desc: 'The name for the job'
-      end
-      get ':id/jobs/artifacts/:ref_name/download',
-        requirements: { ref_name: /.+/ } do
-        authorize_read_builds!
-
-        builds = user_project.latest_successful_builds_for(params[:ref_name])
-        latest_build = builds.find_by!(name: params[:job])
-
-        present_artifacts!(latest_build.artifacts_file)
-      end
-
       # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
       #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
       #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
@@ -112,7 +81,7 @@ module API
       get ':id/jobs/:job_id/trace' do
         authorize_read_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
 
         header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
         content_type 'text/plain'
@@ -131,7 +100,7 @@ module API
       post ':id/jobs/:job_id/cancel' do
         authorize_update_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
         authorize!(:update_build, build)
 
         build.cancel
@@ -148,7 +117,7 @@ module API
       post ':id/jobs/:job_id/retry' do
         authorize_update_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
         authorize!(:update_build, build)
         return forbidden!('Job is not retryable') unless build.retryable?
 
@@ -166,7 +135,7 @@ module API
       post ':id/jobs/:job_id/erase' do
         authorize_update_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
         authorize!(:update_build, build)
         return forbidden!('Job is not erasable!') unless build.erasable?
 
@@ -174,25 +143,6 @@ module API
         present build, with: Entities::Job
       end
 
-      desc 'Keep the artifacts to prevent them from being deleted' do
-        success Entities::Job
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      post ':id/jobs/:job_id/artifacts/keep' do
-        authorize_update_builds!
-
-        build = get_build!(params[:job_id])
-        authorize!(:update_build, build)
-        return not_found!(build) unless build.artifacts?
-
-        build.keep_artifacts!
-
-        status 200
-        present build, with: Entities::Job
-      end
-
       desc 'Trigger a manual job' do
         success Entities::Job
         detail 'This feature was added in GitLab 8.11'
@@ -203,7 +153,7 @@ module API
       post ":id/jobs/:job_id/play" do
         authorize_read_builds!
 
-        build = get_build!(params[:job_id])
+        build = find_build!(params[:job_id])
 
         authorize!(:update_build, build)
         bad_request!("Unplayable Job") unless build.playable?
@@ -216,14 +166,6 @@ module API
     end
 
     helpers do
-      def find_build(id)
-        user_project.builds.find_by(id: id.to_i)
-      end
-
-      def get_build!(id)
-        find_build(id) || not_found!
-      end
-
       def filter_builds(builds, scope)
         return builds if scope.nil? || scope.empty?
 
@@ -234,14 +176,6 @@ module API
 
         builds.where(status: available_statuses && scope)
       end
-
-      def authorize_read_builds!
-        authorize! :read_build, user_project
-      end
-
-      def authorize_update_builds!
-        authorize! :update_build, user_project
-      end
     end
   end
 end
diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb
new file mode 100644
index 00000000000..f5ff95e3eb3
--- /dev/null
+++ b/lib/banzai/commit_renderer.rb
@@ -0,0 +1,11 @@
+module Banzai
+  module CommitRenderer
+    ATTRIBUTES = [:description, :title].freeze
+
+    def self.render(commits, project, user = nil)
+      obj_renderer = ObjectRenderer.new(project, user)
+
+      ATTRIBUTES.each { |attr| obj_renderer.render(commits, attr) }
+    end
+  end
+end
diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb
index 8e7084f2543..47151626208 100644
--- a/lib/banzai/filter/table_of_contents_filter.rb
+++ b/lib/banzai/filter/table_of_contents_filter.rb
@@ -22,40 +22,94 @@ module Banzai
         result[:toc] = ""
 
         headers = Hash.new(0)
+        header_root = current_header = HeaderNode.new
 
         doc.css('h1, h2, h3, h4, h5, h6').each do |node|
-          text = node.text
+          if header_content = node.children.first
+            id = node
+              .text
+              .downcase
+              .gsub(PUNCTUATION_REGEXP, '') # remove punctuation
+              .tr(' ', '-') # replace spaces with dash
+              .squeeze('-') # replace multiple dashes with one
 
-          id = text.downcase
-          id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation
-          id.tr!(' ', '-') # replace spaces with dash
-          id.squeeze!('-') # replace multiple dashes with one
+            uniq = headers[id] > 0 ? "-#{headers[id]}" : ''
+            headers[id] += 1
+            href = "#{id}#{uniq}"
 
-          uniq = (headers[id] > 0) ? "-#{headers[id]}" : ''
-          headers[id] += 1
+            current_header = HeaderNode.new(node: node, href: href, previous_header: current_header)
 
-          if header_content = node.children.first
-            # namespace detection will be automatically handled via javascript (see issue #22781)
-            namespace = "user-content-"
-            href = "#{id}#{uniq}"
-            push_toc(href, text)
-            header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href))
+            header_content.add_previous_sibling(anchor_tag(href))
           end
         end
 
-        result[:toc] = %Q{<ul class="section-nav">\n#{result[:toc]}</ul>} unless result[:toc].empty?
+        push_toc(header_root.children, root: true)
 
         doc
       end
 
       private
 
-      def anchor_tag(id, href)
-        %Q{<a id="#{id}" class="anchor" href="##{href}" aria-hidden="true"></a>}
+      def anchor_tag(href)
+        %Q{<a id="user-content-#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>}
       end
 
-      def push_toc(href, text)
-        result[:toc] << %Q{<li><a href="##{href}">#{text}</a></li>\n}
+      def push_toc(children, root: false)
+        return if children.empty?
+
+        klass = ' class="section-nav"' if root
+
+        result[:toc] << "<ul#{klass}>"
+        children.each { |child| push_anchor(child) }
+        result[:toc] << '</ul>'
+      end
+
+      def push_anchor(header_node)
+        result[:toc] << %Q{<li><a href="##{header_node.href}">#{header_node.text}</a>}
+        push_toc(header_node.children)
+        result[:toc] << '</li>'
+      end
+
+      class HeaderNode
+        attr_reader :node, :href, :parent, :children
+
+        def initialize(node: nil, href: nil, previous_header: nil)
+          @node = node
+          @href = href
+          @children = []
+
+          @parent = find_parent(previous_header)
+          @parent.children.push(self) if @parent
+        end
+
+        def level
+          return 0 unless node
+
+          @level ||= node.name[1].to_i
+        end
+
+        def text
+          return '' unless node
+
+          @text ||= node.text
+        end
+
+        private
+
+        def find_parent(previous_header)
+          return unless previous_header
+
+          if level == previous_header.level
+            parent = previous_header.parent
+          elsif level > previous_header.level
+            parent = previous_header
+          else
+            parent = previous_header
+            parent = parent.parent while parent.level >= level
+          end
+
+          parent
+        end
       end
     end
   end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
index 2196a92474c..e40556e869c 100644
--- a/lib/banzai/object_renderer.rb
+++ b/lib/banzai/object_renderer.rb
@@ -38,7 +38,7 @@ module Banzai
       objects.each_with_index do |object, index|
         redacted_data = redacted[index]
         object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) # rubocop:disable GitlabSecurity/PublicSend
-        object.user_visible_reference_count = redacted_data[:visible_reference_count]
+        object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count)
       end
     end
 
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 95d82d17658..ceca9296851 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -36,6 +36,10 @@ module Banzai
     # The context to use is managed by the object and cannot be changed.
     # Use #render, passing it the field text, if a custom rendering is needed.
     def self.render_field(object, field)
+      unless object.respond_to?(:cached_markdown_fields)
+        return cacheless_render_field(object, field)
+      end
+
       object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field)
 
       object.cached_html_for(field)
diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb
index c6fa928d565..823e8e9a9c4 100644
--- a/lib/github/representation/branch.rb
+++ b/lib/github/representation/branch.rb
@@ -41,7 +41,7 @@ module Github
 
       def remove!(name)
         repository.delete_branch(name)
-      rescue Rugged::ReferenceError => e
+      rescue Gitlab::Git::Repository::DeleteBranchError => e
         Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}")
       end
 
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index 4714ab18cc1..b4012ebbb99 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -67,10 +67,14 @@ module Gitlab
       def protection_values
         protection_options.values
       end
+
+      def human_access(access)
+        options_with_owner.key(access)
+      end
     end
 
     def human_access
-      Gitlab::Access.options_with_owner.key(access_field)
+      Gitlab::Access.human_access(access_field)
     end
 
     def owner?
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index 2e073334abc..22941d48edf 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -1,129 +1,129 @@
 module Gitlab
-  module Ci::Build::Artifacts
-    class Metadata
-      ##
-      # Class that represents an entry (path and metadata) to a file or
-      # directory in GitLab CI Build Artifacts binary file / archive
-      #
-      # This is IO-operations safe class, that does similar job to
-      # Ruby's Pathname but without the risk of accessing filesystem.
-      #
-      # This class is working only with UTF-8 encoded paths.
-      #
-      class Entry
-        attr_reader :path, :entries
-        attr_accessor :name
-
-        def initialize(path, entries)
-          @path = path.dup.force_encoding('UTF-8')
-          @entries = entries
-
-          if path.include?("\0")
-            raise ArgumentError, 'Path contains zero byte character!'
-          end
+  module Ci
+    module Build
+      module Artifacts
+        class Metadata
+          ##
+          # Class that represents an entry (path and metadata) to a file or
+          # directory in GitLab CI Build Artifacts binary file / archive
+          #
+          # This is IO-operations safe class, that does similar job to
+          # Ruby's Pathname but without the risk of accessing filesystem.
+          #
+          # This class is working only with UTF-8 encoded paths.
+          #
+          class Entry
+            attr_reader :entries
+            attr_accessor :name
+
+            def initialize(path, entries)
+              @entries = entries
+              @path = Artifacts::Path.new(path)
+            end
+
+            delegate :empty?, to: :children
+
+            def directory?
+              blank_node? || @path.directory?
+            end
+
+            def file?
+              !directory?
+            end
+
+            def blob
+              return unless file?
+
+              @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil)
+            end
+
+            def has_parent?
+              nodes > 0
+            end
+
+            def parent
+              return nil unless has_parent?
+              self.class.new(@path.to_s.chomp(basename), @entries)
+            end
+
+            def basename
+              (directory? && !blank_node?) ? name + '/' : name
+            end
+
+            def name
+              @name || @path.name
+            end
+
+            def children
+              return [] unless directory?
+              return @children if @children
+
+              child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$}
+              @children = select_entries { |path| path =~ child_pattern }
+            end
+
+            def directories(opts = {})
+              return [] unless directory?
+              dirs = children.select(&:directory?)
+              return dirs unless has_parent? && opts[:parent]
+
+              dotted_parent = parent
+              dotted_parent.name = '..'
+              dirs.prepend(dotted_parent)
+            end
+
+            def files
+              return [] unless directory?
+              children.select(&:file?)
+            end
+
+            def metadata
+              @entries[@path.to_s] || {}
+            end
+
+            def nodes
+              @path.nodes + (file? ? 1 : 0)
+            end
+
+            def blank_node?
+              @path.to_s.empty? # "" is considered to be './'
+            end
+
+            def exists?
+              blank_node? || @entries.include?(@path.to_s)
+            end
+
+            def total_size
+              descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}}
+              entries.sum do |path, entry|
+                (entry[:size] if path =~ descendant_pattern).to_i
+              end
+            end
+
+            def path
+              @path.to_s
+            end
+
+            def to_s
+              @path.to_s
+            end
+
+            def ==(other)
+              path == other.path && @entries == other.entries
+            end
+
+            def inspect
+              "#{self.class.name}: #{self}"
+            end
 
-          unless path.valid_encoding?
-            raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+            private
+
+            def select_entries
+              selected = @entries.select { |path, _metadata| yield path }
+              selected.map { |path, _metadata| self.class.new(path, @entries) }
+            end
           end
         end
-
-        delegate :empty?, to: :children
-
-        def directory?
-          blank_node? || @path.end_with?('/')
-        end
-
-        def file?
-          !directory?
-        end
-
-        def blob
-          return unless file?
-
-          @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil)
-        end
-
-        def has_parent?
-          nodes > 0
-        end
-
-        def parent
-          return nil unless has_parent?
-          self.class.new(@path.chomp(basename), @entries)
-        end
-
-        def basename
-          (directory? && !blank_node?) ? name + '/' : name
-        end
-
-        def name
-          @name || @path.split('/').last.to_s
-        end
-
-        def children
-          return [] unless directory?
-          return @children if @children
-
-          child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
-          @children = select_entries { |path| path =~ child_pattern }
-        end
-
-        def directories(opts = {})
-          return [] unless directory?
-          dirs = children.select(&:directory?)
-          return dirs unless has_parent? && opts[:parent]
-
-          dotted_parent = parent
-          dotted_parent.name = '..'
-          dirs.prepend(dotted_parent)
-        end
-
-        def files
-          return [] unless directory?
-          children.select(&:file?)
-        end
-
-        def metadata
-          @entries[@path] || {}
-        end
-
-        def nodes
-          @path.count('/') + (file? ? 1 : 0)
-        end
-
-        def blank_node?
-          @path.empty? # "" is considered to be './'
-        end
-
-        def exists?
-          blank_node? || @entries.include?(@path)
-        end
-
-        def total_size
-          descendant_pattern = %r{^#{Regexp.escape(@path)}}
-          entries.sum do |path, entry|
-            (entry[:size] if path =~ descendant_pattern).to_i
-          end
-        end
-
-        def to_s
-          @path
-        end
-
-        def ==(other)
-          @path == other.path && @entries == other.entries
-        end
-
-        def inspect
-          "#{self.class.name}: #{@path}"
-        end
-
-        private
-
-        def select_entries
-          selected = @entries.select { |path, _metadata| yield path }
-          selected.map { |path, _metadata| self.class.new(path, @entries) }
-        end
       end
     end
   end
diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb
new file mode 100644
index 00000000000..9cd9b36c5f8
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/path.rb
@@ -0,0 +1,51 @@
+module Gitlab
+  module Ci
+    module Build
+      module Artifacts
+        class Path
+          def initialize(path)
+            @path = path.dup.force_encoding('UTF-8')
+          end
+
+          def valid?
+            nonzero? && utf8?
+          end
+
+          def directory?
+            @path.end_with?('/')
+          end
+
+          def name
+            @path.split('/').last.to_s
+          end
+
+          def nodes
+            @path.count('/')
+          end
+
+          def to_s
+            @path.tap do |path|
+              unless nonzero?
+                raise ArgumentError, 'Path contains zero byte character!'
+              end
+
+              unless utf8?
+                raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+              end
+            end
+          end
+
+          private
+
+          def nonzero?
+            @path.exclude?("\0")
+          end
+
+          def utf8?
+            @path.valid_encoding?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 8ddc91e341d..7b3483a7f96 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -22,10 +22,10 @@ module Gitlab
 
       # return message if message type is binary
       detect = CharlockHolmes::EncodingDetector.detect(message)
-      return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+      return message.force_encoding("BINARY") if detect_binary?(message, detect)
 
-      # force detected encoding if we have sufficient confidence.
       if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
+        # force detected encoding if we have sufficient confidence.
         message.force_encoding(detect[:encoding])
       end
 
@@ -36,6 +36,19 @@ module Gitlab
       "--broken encoding: #{encoding}"
     end
 
+    def detect_binary?(data, detect = nil)
+      detect ||= CharlockHolmes::EncodingDetector.detect(data)
+      detect && detect[:type] == :binary && detect[:confidence] == 100
+    end
+
+    def detect_libgit2_binary?(data)
+      # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
+      # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
+      # which is what we use below to keep a consistent behavior.
+      detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
+      detect && detect[:type] == :binary
+    end
+
     def encode_utf8(message)
       detect = CharlockHolmes::EncodingDetector.detect(message)
       if detect && detect[:encoding]
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 7780f4e4d4f..8d96826f6ee 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -42,14 +42,6 @@ module Gitlab
           end
         end
 
-        def binary?(data)
-          # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
-          # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
-          # which is what we use below to keep a consistent behavior.
-          detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
-          detect && detect[:type] == :binary
-        end
-
         # Returns an array of Blob instances, specified in blob_references as
         # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the
         # full blob contents are returned. If blob_size_limit >= 0 then each blob will
@@ -65,6 +57,10 @@ module Gitlab
           end
         end
 
+        def binary?(data)
+          EncodingHelper.detect_libgit2_binary?(data)
+        end
+
         private
 
         # Recursive search of blob id by path
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index ce3d65062e8..a23c8cf0dd1 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -116,6 +116,15 @@ module Gitlab
 
           filtered_opts
         end
+
+        # Return a binary diff message like:
+        #
+        # "Binary files a/file/path and b/file/path differ\n"
+        # This is used when we detect that a diff is binary
+        # using CharlockHolmes when Rugged treats it as text.
+        def binary_message(old_path, new_path)
+          "Binary files #{old_path} and #{new_path} differ\n"
+        end
       end
 
       def initialize(raw_diff, expanded: true)
@@ -190,6 +199,13 @@ module Gitlab
         @collapsed = true
       end
 
+      def json_safe_diff
+        return @diff unless detect_binary?(@diff)
+
+        # the diff is binary, let's make a message for it
+        Diff.binary_message(@old_path, @new_path)
+      end
+
       private
 
       def init_from_rugged(rugged)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 75d4efc0bc5..efa13590a2c 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -18,6 +18,7 @@ module Gitlab
       InvalidBlobName = Class.new(StandardError)
       InvalidRef = Class.new(StandardError)
       GitError = Class.new(StandardError)
+      DeleteBranchError = Class.new(StandardError)
 
       class << self
         # Unlike `new`, `create` takes the storage path, not the storage name
@@ -653,10 +654,16 @@ module Gitlab
       end
 
       # Delete the specified branch from the repository
-      #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def delete_branch(branch_name)
-        rugged.branches.delete(branch_name)
+        gitaly_migrate(:delete_branch) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.delete_branch(branch_name)
+          else
+            rugged.branches.delete(branch_name)
+          end
+        end
+      rescue Rugged::ReferenceError, CommandError => e
+        raise DeleteBranchError, e
       end
 
       def delete_refs(*ref_names)
@@ -681,15 +688,14 @@ module Gitlab
       # Examples:
       #   create_branch("feature")
       #   create_branch("other-feature", "master")
-      #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476
       def create_branch(ref, start_point = "HEAD")
-        rugged_ref = rugged.branches.create(ref, start_point)
-        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
-      rescue Rugged::ReferenceError => e
-        raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
-        raise InvalidRef.new("Invalid reference #{start_point}")
+        gitaly_migrate(:create_branch) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.create_branch(ref, start_point)
+          else
+            rugged_create_branch(ref, start_point)
+          end
+        end
       end
 
       # Delete the specified remote from this repository.
@@ -1226,6 +1232,15 @@ module Gitlab
         false
       end
 
+      def rugged_create_branch(ref, start_point)
+        rugged_ref = rugged.branches.create(ref, start_point)
+        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+      rescue Rugged::ReferenceError => e
+        raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
+        raise InvalidRef.new("Invalid reference #{start_point}")
+      end
+
       def gitaly_copy_gitattributes(revision)
         gitaly_repository_client.apply_gitattributes(revision)
       end
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index a1a25cf2079..8ef873d5848 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -79,7 +79,7 @@ module Gitlab
       end
 
       def find_branch(branch_name)
-        request = Gitaly::DeleteBranchRequest.new(
+        request = Gitaly::FindBranchRequest.new(
           repository: @gitaly_repo,
           name: GitalyClient.encode(branch_name)
         )
@@ -92,6 +92,40 @@ module Gitlab
         Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit)
       end
 
+      def create_branch(ref, start_point)
+        request = Gitaly::CreateBranchRequest.new(
+          repository: @gitaly_repo,
+          name: GitalyClient.encode(ref),
+          start_point: GitalyClient.encode(start_point)
+        )
+
+        response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request)
+
+        case response.status
+        when :OK
+          branch = response.branch
+          target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit)
+          Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit)
+        when :ERR_INVALID
+          invalid_ref!("Invalid ref name")
+        when :ERR_EXISTS
+          invalid_ref!("Branch #{ref} already exists")
+        when :ERR_INVALID_START_POINT
+          invalid_ref!("Invalid reference #{start_point}")
+        else
+          raise "Unknown response status: #{response.status}"
+        end
+      end
+
+      def delete_branch(branch_name)
+        request = Gitaly::DeleteBranchRequest.new(
+          repository: @gitaly_repo,
+          name: GitalyClient.encode(branch_name)
+        )
+
+        GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request)
+      end
+
       private
 
       def consume_refs_response(response)
@@ -163,6 +197,10 @@ module Gitlab
 
         Gitlab::Git::Commit.decorate(@repository, hash)
       end
+
+      def invalid_ref!(message)
+        raise Gitlab::Git::Repository::InvalidRef.new(message)
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 373062b354b..b8c07460ebb 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -166,7 +166,7 @@ module Gitlab
 
       def remove_branch(name)
         project.repository.delete_branch(name)
-      rescue Rugged::ReferenceError
+      rescue Gitlab::Git::Repository::DeleteBranchFailed
         errors << { type: :remove_branch, name: name }
       end
 
diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb
index 39180dc17d9..3bf27b37ae6 100644
--- a/lib/gitlab/ldap/user.rb
+++ b/lib/gitlab/ldap/user.rb
@@ -36,7 +36,7 @@ module Gitlab
       end
 
       def find_by_email
-        ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email?
+        ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email)
       end
 
       def update_user_attributes
@@ -60,7 +60,7 @@ module Gitlab
         ldap_config.block_auto_created_users
       end
 
-      def sync_email_from_provider?
+      def sync_profile_from_provider?
         true
       end
 
diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb
index 7d6911a1ab3..1f331b1e91d 100644
--- a/lib/gitlab/o_auth/auth_hash.rb
+++ b/lib/gitlab/o_auth/auth_hash.rb
@@ -32,8 +32,21 @@ module Gitlab
         @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase)
       end
 
-      def has_email?
-        get_info(:email).present?
+      def location
+        location = get_info(:address)
+        if location.is_a?(Hash)
+          [location.locality.presence, location.country.presence].compact.join(', ')
+        else
+          location
+        end
+      end
+
+      def has_attribute?(attribute)
+        if attribute == :location
+          get_info(:address).present?
+        else
+          get_info(attribute).present?
+        end
       end
 
       private
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index e8330917e91..7704bf715e4 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -12,7 +12,7 @@ module Gitlab
 
       def initialize(auth_hash)
         self.auth_hash = auth_hash
-        update_email
+        update_profile if sync_profile_from_provider?
       end
 
       def persisted?
@@ -184,20 +184,30 @@ module Gitlab
         }
       end
 
-      def sync_email_from_provider?
-        auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s
+      def sync_profile_from_provider?
+        providers = Gitlab.config.omniauth.sync_profile_from_provider
+
+        if providers.is_a?(Array)
+          providers.include?(auth_hash.provider)
+        else
+          providers
+        end
       end
 
-      def update_email
-        if auth_hash.has_email? && sync_email_from_provider?
-          if persisted?
-            gl_user.skip_reconfirmation!
-            gl_user.email = auth_hash.email
-          end
+      def update_profile
+        user_synced_attributes_metadata = gl_user.user_synced_attributes_metadata || gl_user.build_user_synced_attributes_metadata
 
-          gl_user.external_email = true
-          gl_user.email_provider = auth_hash.provider
+        UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key|
+          if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key)
+            gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend
+            user_synced_attributes_metadata.set_attribute_synced(key, true)
+          else
+            user_synced_attributes_metadata.set_attribute_synced(key, false)
+          end
         end
+
+        user_synced_attributes_metadata.provider = auth_hash.provider
+        gl_user.user_synced_attributes_metadata = user_synced_attributes_metadata
       end
 
       def log
diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb
index 8a7cc690046..0f323a9e8b2 100644
--- a/lib/gitlab/saml/user.rb
+++ b/lib/gitlab/saml/user.rb
@@ -40,7 +40,7 @@ module Gitlab
       end
 
       def find_by_email
-        if auth_hash.has_email?
+        if auth_hash.has_attribute?(:email)
           user = ::User.find_by(email: auth_hash.email.downcase)
           user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user
           user
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index c81dc7e30d0..703adae12cb 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -9,7 +9,7 @@ module Gitlab
     end
 
     def self.valid?(url)
-      return false unless url
+      return false unless url.present?
 
       Addressable::URI.parse(url.strip)
 
@@ -19,7 +19,12 @@ module Gitlab
     end
 
     def initialize(url, credentials: nil)
-      @url = Addressable::URI.parse(url.strip)
+      @url = Addressable::URI.parse(url.to_s.strip)
+
+      %i[user password].each do |symbol|
+        credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol)
+      end
+
       @credentials = credentials
     end
 
@@ -29,13 +34,13 @@ module Gitlab
 
     def masked_url
       url = @url.dup
-      url.password = "*****" unless url.password.nil?
-      url.user = "*****" unless url.user.nil?
+      url.password = "*****" if url.password.present?
+      url.user = "*****" if url.user.present?
       url.to_s
     end
 
     def credentials
-      @credentials ||= { user: @url.user, password: @url.password }
+      @credentials ||= { user: @url.user.presence, password: @url.password.presence }
     end
 
     def full_url
@@ -47,8 +52,10 @@ module Gitlab
     def generate_full_url
       return @url unless valid_credentials?
       @full_url = @url.dup
-      @full_url.user = credentials[:user]
+
       @full_url.password = credentials[:password]
+      @full_url.user = credentials[:user]
+
       @full_url
     end
 
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index e5ad9b5a40c..7a94af2f8f1 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -121,10 +121,10 @@ module Gitlab
         ]
       end
 
-      def send_artifacts_entry(build, entry)
+      def send_artifacts_entry(build, path)
         params = {
           'Archive' => build.artifacts_file.path,
-          'Entry' => Base64.encode64(entry.path)
+          'Entry' => Base64.encode64(path.to_s)
         }
 
         [