Merge remote-tracking branch 'upstream/master' into 30634-protected-pipeline

* upstream/master: (638 commits)
  Simplify background migrations stealing code
  Expire cached user IDs that can see the performance after 5 minutes
  Promote visibility level helpers from Group to Namespace
  Fix off-by-one error in background migration retries
  Recover from all exceptions when stealing bg migration
  Fix label creation from new list for subgroup projects
  move click handler to button. when on the icon it wasn't triggered in firefox
  Fix incorrect AWS ELB metrics.
  Fix wrong link to docs in docs styleguide
  Update issue-related docs
  Refactor groups docs
  Add subgroups limitations to Pages docs
  Update Google launcher details
  Split docs on IP whitelist for monitoring access
  Update health check docs
  Bump fog-core to 1.44.3 and fog providers' plugins to latest
  Introduce have_gitlab_http_status
  Remove Repository#search_files
  Update Pipeline's badge count in Merge Request and Commits view to match real-time content
  Fixes the user order being overriden in the autocomplete controller
  ...

120 files changed, 1550 insertions, 874 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index d767af36e8e..efcf0976a81 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -2,6 +2,8 @@ module API
   class API < Grape::API
     include APIGuard
 
+    allow_access_with_scope :api
+
     version %w(v3 v4), using: :path
 
     version 'v3', using: :path do
@@ -44,7 +46,6 @@ module API
       mount ::API::V3::Variables
     end
 
-    before { allow_access_with_scope :api }
     before { header['X-Frame-Options'] = 'SAMEORIGIN' }
     before { Gitlab::I18n.locale = current_user&.preferred_language }
 
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 9fcf04efa38..0d2d71e336a 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -23,6 +23,23 @@ module API
       install_error_responders(base)
     end
 
+    class_methods do
+      # Set the authorization scope(s) allowed for an API endpoint.
+      #
+      # A call to this method maps the given scope(s) to the current API
+      # endpoint class. If this method is called multiple times on the same class,
+      # the scopes are all aggregated.
+      def allow_access_with_scope(scopes, options = {})
+        Array(scopes).each do |scope|
+          allowed_scopes << Scope.new(scope, options)
+        end
+      end
+
+      def allowed_scopes
+        @scopes ||= []
+      end
+    end
+
     # Helper Methods for Grape Endpoint
     module HelperMethods
       # Invokes the doorkeeper guard.
@@ -47,7 +64,7 @@ module API
         access_token = find_access_token
         return nil unless access_token
 
-        case AccessTokenValidationService.new(access_token).validate(scopes: scopes)
+        case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes)
         when AccessTokenValidationService::INSUFFICIENT_SCOPE
           raise InsufficientScopeError.new(scopes)
 
@@ -74,18 +91,6 @@ module API
         @current_user
       end
 
-      # Set the authorization scope(s) allowed for the current request.
-      #
-      # Note: A call to this method adds to any previous scopes in place. This is done because
-      # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then
-      # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the
-      # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they
-      # need to be stored.
-      def allow_access_with_scope(*scopes)
-        @scopes ||= []
-        @scopes.concat(scopes.map(&:to_s))
-      end
-
       private
 
       def find_user_by_authentication_token(token_string)
@@ -96,7 +101,7 @@ module API
         access_token = PersonalAccessToken.active.find_by_token(token_string)
         return unless access_token
 
-        if AccessTokenValidationService.new(access_token).include_any_scope?(scopes)
+        if AccessTokenValidationService.new(access_token, request: request).include_any_scope?(scopes)
           User.find(access_token.user_id)
         end
       end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index cef5a0abe12..09a88869063 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -112,6 +112,7 @@ module API
       expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds, as: :public_jobs
+      expose :ci_config_path
       expose :shared_with_groups do |project, options|
         SharedGroup.represent(project.project_group_links.all, options)
       end
@@ -254,7 +255,7 @@ module API
 
     class ProjectEntity < Grape::Entity
       expose :id, :iid
-      expose(:project_id) { |entity| entity.project.id }
+      expose(:project_id) { |entity| entity&.project.try(:id) }
       expose :title, :description
       expose :state, :created_at, :updated_at
     end
@@ -266,7 +267,12 @@ module API
       expose :deleted_file?, as: :deleted_file
     end
 
-    class Milestone < ProjectEntity
+    class Milestone < Grape::Entity
+      expose :id, :iid
+      expose(:project_id) { |entity| entity&.project_id }
+      expose(:group_id) { |entity| entity&.group_id }
+      expose :title, :description
+      expose :state, :created_at, :updated_at
       expose :due_date
       expose :start_date
     end
@@ -308,12 +314,35 @@ module API
       expose :id
     end
 
+    class MergeRequestSimple < ProjectEntity
+      expose :title
+      expose :web_url do |merge_request, options|
+        Gitlab::UrlBuilder.build(merge_request)
+      end
+    end
+
     class MergeRequestBasic < ProjectEntity
       expose :target_branch, :source_branch
-      expose :upvotes, :downvotes
+      expose :upvotes do |merge_request, options|
+        if options[:issuable_metadata]
+          options[:issuable_metadata][merge_request.id].upvotes
+        else
+          merge_request.upvotes
+        end
+      end
+      expose :downvotes do |merge_request, options|
+        if options[:issuable_metadata]
+          options[:issuable_metadata][merge_request.id].downvotes
+        else
+          merge_request.downvotes
+        end
+      end
       expose :author, :assignee, using: Entities::UserBasic
       expose :source_project_id, :target_project_id
-      expose :label_names, as: :labels
+      expose :labels do |merge_request, options|
+        # Avoids an N+1 query since labels are preloaded
+        merge_request.labels.map(&:title).sort
+      end
       expose :work_in_progress?, as: :work_in_progress
       expose :milestone, using: Entities::Milestone
       expose :merge_when_pipeline_succeeds
@@ -434,7 +463,7 @@ module API
         target_url    = "namespace_project_#{target_type}_url"
         target_anchor = "note_#{todo.note_id}" if todo.note_id?
 
-        Gitlab::Application.routes.url_helpers.public_send(target_url,
+        Gitlab::Routing.url_helpers.public_send(target_url,
           todo.project.namespace, todo.project, todo.target, anchor: target_anchor)
       end
 
@@ -502,12 +531,20 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          project.project_members.find_by(user_id: options[:current_user].id)
+          if options.key?(:project_members)
+            (options[:project_members] || []).find { |member| member.source_id == project.id }
+          else
+            project.project_members.find_by(user_id: options[:current_user].id)
+          end
         end
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            project.group.group_members.find_by(user_id: options[:current_user].id)
+            if options.key?(:group_members)
+              (options[:group_members] || []).find { |member| member.source_id == project.namespace_id }
+            else
+              project.group.group_members.find_by(user_id: options[:current_user].id)
+            end
           end
         end
       end
@@ -584,7 +621,8 @@ module API
       expose :id
       expose :default_projects_limit
       expose :signup_enabled
-      expose :signin_enabled
+      expose :password_authentication_enabled
+      expose :password_authentication_enabled, as: :signin_enabled
       expose :gravatar_enabled
       expose :sign_in_text
       expose :after_sign_up_text
@@ -831,7 +869,7 @@ module API
       end
 
       class Cache < Grape::Entity
-        expose :key, :untracked, :paths
+        expose :key, :untracked, :paths, :policy
       end
 
       class Credentials < Grape::Entity
@@ -874,5 +912,11 @@ module API
         expose :dependencies, using: Dependency
       end
     end
+
+    class UserAgentDetail < Grape::Entity
+      expose :user_agent
+      expose :ip_address
+      expose :submitted, as: :akismet_submitted
+    end
   end
 end
diff --git a/lib/api/features.rb b/lib/api/features.rb
index 21745916463..9385c6ca174 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -14,14 +14,12 @@ module API
         end
       end
 
-      def gate_target(params)
-        if params[:feature_group]
-          Feature.group(params[:feature_group])
-        elsif params[:user]
-          User.find_by_username(params[:user])
-        else
-          gate_value(params)
-        end
+      def gate_targets(params)
+        targets = []
+        targets << Feature.group(params[:feature_group]) if params[:feature_group]
+        targets << User.find_by_username(params[:user]) if params[:user]
+
+        targets
       end
     end
 
@@ -42,18 +40,25 @@ module API
         requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
         optional :feature_group, type: String, desc: 'A Feature group name'
         optional :user, type: String, desc: 'A GitLab username'
-        mutually_exclusive :feature_group, :user
       end
       post ':name' do
         feature = Feature.get(params[:name])
-        target = gate_target(params)
+        targets = gate_targets(params)
         value = gate_value(params)
 
         case value
         when true
-          feature.enable(target)
+          if targets.present?
+            targets.each { |target| feature.enable(target) }
+          else
+            feature.enable
+          end
         when false
-          feature.disable(target)
+          if targets.present?
+            targets.each { |target| feature.disable(target) }
+          else
+            feature.disable
+          end
         else
           feature.enable_percentage_of_time(value)
         end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 2c73a6fdc4e..0f4791841d2 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -268,6 +268,7 @@ module API
       finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
       finder_params[:archived] = params[:archived]
       finder_params[:search] = params[:search] if params[:search]
+      finder_params[:user] = params.delete(:user) if params[:user]
       finder_params
     end
 
@@ -313,7 +314,7 @@ module API
 
     def present_artifacts!(artifacts_file)
       return not_found! unless artifacts_file.exists?
-  
+
       if artifacts_file.file_storage?
         present_file!(artifacts_file.path, artifacts_file.filename)
       else
@@ -342,8 +343,8 @@ module API
     def initial_current_user
       return @initial_current_user if defined?(@initial_current_user)
       Gitlab::Auth::UniqueIpsLimiter.limit_user! do
-        @initial_current_user ||= find_user_by_private_token(scopes: @scopes)
-        @initial_current_user ||= doorkeeper_guard(scopes: @scopes)
+        @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint)
+        @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint)
         @initial_current_user ||= find_user_from_warden
 
         unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed?
@@ -407,5 +408,22 @@ module API
 
       exception.status == 500
     end
+
+    # An array of scopes that were registered (using `allow_access_with_scope`)
+    # for the current endpoint class. It also returns scopes registered on
+    # `API::API`, since these are meant to apply to all API routes.
+    def scopes_registered_for_endpoint
+      @scopes_registered_for_endpoint ||=
+        begin
+          endpoint_classes = [options[:for].presence, ::API::API].compact
+          endpoint_classes.reduce([]) do |memo, endpoint|
+            if endpoint.respond_to?(:allowed_scopes)
+              memo.concat(endpoint.allowed_scopes)
+            else
+              memo
+            end
+          end
+        end
+    end
   end
 end
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 5e9cf5e68b1..ecb79317093 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -1,6 +1,11 @@
 module API
   module Helpers
     module InternalHelpers
+      SSH_GITALY_FEATURES = {
+        'git-receive-pack' => :ssh_receive_pack,
+        'git-upload-pack' => :ssh_upload_pack
+      }.freeze
+
       def wiki?
         set_project unless defined?(@wiki)
         @wiki
@@ -10,7 +15,7 @@ module API
         set_project unless defined?(@project)
         @project
       end
-      
+
       def redirected_path
         @redirected_path
       end
@@ -54,15 +59,33 @@ module API
         Gitlab::GlRepository.gl_repository(project, wiki?)
       end
 
-      # Return the repository full path so that gitlab-shell has it when
-      # handling ssh commands
-      def repository_path
+      # Return the repository depending on whether we want the wiki or the
+      # regular repository
+      def repository
         if wiki?
-          project.wiki.repository.path_to_repo
+          project.wiki.repository
         else
-          project.repository.path_to_repo
+          project.repository
         end
       end
+
+      # Return the repository full path so that gitlab-shell has it when
+      # handling ssh commands
+      def repository_path
+        repository.path_to_repo
+      end
+
+      # Return the Gitaly Address if it is enabled
+      def gitaly_payload(action)
+        feature = SSH_GITALY_FEATURES[action]
+        return unless feature && Gitlab::GitalyClient.feature_enabled?(feature)
+
+        {
+          repository: repository.gitaly_repository,
+          address: Gitlab::GitalyClient.address(project.repository_storage),
+          token: Gitlab::GitalyClient.token(project.repository_storage)
+        }
+      end
     end
   end
 end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index f1c79970ba4..465363da582 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -47,7 +47,8 @@ module API
         {
           status: true,
           gl_repository: gl_repository,
-          repository_path: repository_path
+          repository_path: repository_path,
+          gitaly: gitaly_payload(params[:action])
         }
       end
 
@@ -100,7 +101,7 @@ module API
       end
 
       get "/broadcast_message" do
-        if message = BroadcastMessage.current.last
+        if message = BroadcastMessage.current&.last
           present message, with: Entities::BroadcastMessage
         else
           {}
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 09dca0dff8b..64be08094ed 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -241,6 +241,22 @@ module API
 
         present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
       end
+
+      desc 'Get the user agent details for an issue' do
+        success Entities::UserAgentDetail
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+      end
+      get ":id/issues/:issue_iid/user_agent_detail" do
+        authenticated_as_admin!
+
+        issue = find_project_issue(params[:issue_iid])
+
+        return not_found!('UserAgentDetail') unless issue.user_agent_detail
+
+        present issue.user_agent_detail, with: Entities::UserAgentDetail
+      end
     end
   end
 end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 1118fc7465b..ac33b2b801c 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -10,6 +10,8 @@ module API
     resource :projects, requirements: { id: %r{[^/]+} } do
       include TimeTrackingEndpoints
 
+      helpers ::Gitlab::IssuableMetadata
+
       helpers do
         def handle_merge_request_errors!(errors)
           if errors[:project_access].any?
@@ -41,9 +43,15 @@ module API
           args[:milestone_title] = args.delete(:milestone)
           args[:label_name] = args.delete(:labels)
 
-          merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations
+          merge_requests = MergeRequestsFinder.new(current_user, args).execute
+                             .reorder(args[:order_by] => args[:sort])
+          merge_requests = paginate(merge_requests)
+                             .preload(:target_project)
+
+          return merge_requests if args[:view] == 'simple'
 
-          merge_requests.reorder(args[:order_by] => args[:sort])
+          merge_requests
+            .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels)
         end
 
         params :optional_params_ce do
@@ -74,6 +82,7 @@ module API
         optional :labels, type: String, desc: 'Comma-separated list of label names'
         optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time'
         optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time'
+        optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request'
         use :pagination
       end
       get ":id/merge_requests" do
@@ -81,7 +90,17 @@ module API
 
         merge_requests = find_merge_requests(project_id: user_project.id)
 
-        present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
+        options = { with: Entities::MergeRequestBasic,
+                    current_user: current_user,
+                    project: user_project }
+
+        if params[:view] == 'simple'
+          options[:with] = Entities::MergeRequestSimple
+        else
+          options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest')
+        end
+
+        present merge_requests, options
       end
 
       desc 'Create a merge request' do
diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb
index 93d89209934..dbeaf9e17ef 100644
--- a/lib/api/pipeline_schedules.rb
+++ b/lib/api/pipeline_schedules.rb
@@ -74,9 +74,10 @@ module API
         optional :active, type: Boolean, desc: 'The activation of pipeline schedule'
       end
       put ':id/pipeline_schedules/:pipeline_schedule_id' do
-        authorize! :update_pipeline_schedule, user_project
+        authorize! :read_pipeline_schedule, user_project
 
         not_found!('PipelineSchedule') unless pipeline_schedule
+        authorize! :update_pipeline_schedule, pipeline_schedule
 
         if pipeline_schedule.update(declared_params(include_missing: false))
           present pipeline_schedule, with: Entities::PipelineScheduleDetails
@@ -92,9 +93,10 @@ module API
         requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
       end
       post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do
-        authorize! :update_pipeline_schedule, user_project
+        authorize! :read_pipeline_schedule, user_project
 
         not_found!('PipelineSchedule') unless pipeline_schedule
+        authorize! :update_pipeline_schedule, pipeline_schedule
 
         if pipeline_schedule.own!(current_user)
           present pipeline_schedule, with: Entities::PipelineScheduleDetails
@@ -110,9 +112,10 @@ module API
         requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
       end
       delete ':id/pipeline_schedules/:pipeline_schedule_id' do
-        authorize! :admin_pipeline_schedule, user_project
+        authorize! :read_pipeline_schedule, user_project
 
         not_found!('PipelineSchedule') unless pipeline_schedule
+        authorize! :admin_pipeline_schedule, pipeline_schedule
 
         status :accepted
         present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index 64efe82a937..3320eadff0d 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -131,6 +131,22 @@ module API
         content_type 'text/plain'
         present snippet.content
       end
+
+      desc 'Get the user agent details for a project snippet' do
+        success Entities::UserAgentDetail
+      end
+      params do
+        requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+      end
+      get ":id/snippets/:snippet_id/user_agent_detail" do
+        authenticated_as_admin!
+
+        snippet = Snippet.find_by!(id: params[:id])
+
+        return not_found!('UserAgentDetail') unless snippet.user_agent_detail
+
+        present snippet.user_agent_detail, with: Entities::UserAgentDetail
+      end
     end
   end
 end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index d0bd64b2972..c459257158d 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -10,6 +10,7 @@ module API
     helpers do
       params :optional_params_ce do
         optional :description, type: String, desc: 'The description of the project'
+        optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`'
         optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
         optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
         optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'
@@ -35,61 +36,86 @@ module API
       params :statistics_params do
         optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
       end
-    end
 
-    resource :projects do
-      helpers do
-        params :collection_params do
-          use :sort_params
-          use :filter_params
-          use :pagination
-
-          optional :simple, type: Boolean, default: false,
-                            desc: 'Return only the ID, URL, name, and path of each project'
-        end
+      params :collection_params do
+        use :sort_params
+        use :filter_params
+        use :pagination
 
-        params :sort_params do
-          optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
-                              default: 'created_at', desc: 'Return projects ordered by field'
-          optional :sort, type: String, values: %w[asc desc], default: 'desc',
-                          desc: 'Return projects sorted in ascending and descending order'
-        end
+        optional :simple, type: Boolean, default: false,
+                          desc: 'Return only the ID, URL, name, and path of each project'
+      end
 
-        params :filter_params do
-          optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
-          optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
-                                desc: 'Limit by visibility'
-          optional :search, type: String, desc: 'Return list of projects matching the search criteria'
-          optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
-          optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
-          optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
-          optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
-          optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
-        end
+      params :sort_params do
+        optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+                            default: 'created_at', desc: 'Return projects ordered by field'
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return projects sorted in ascending and descending order'
+      end
 
-        params :create_params do
-          optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
-          optional :import_url, type: String, desc: 'URL from which the project is imported'
-        end
+      params :filter_params do
+        optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
+                              desc: 'Limit by visibility'
+        optional :search, type: String, desc: 'Return list of projects matching the search criteria'
+        optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
+        optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
+        optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
+        optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+        optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+      end
+
+      params :create_params do
+        optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
+        optional :import_url, type: String, desc: 'URL from which the project is imported'
+      end
 
-        def present_projects(options = {})
-          projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
-          projects = reorder_projects(projects)
-          projects = projects.with_statistics if params[:statistics]
-          projects = projects.with_issues_enabled if params[:with_issues_enabled]
-          projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
-
-          options = options.reverse_merge(
-            with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
-            statistics: params[:statistics],
-            current_user: current_user
-          )
-          options[:with] = Entities::BasicProjectDetails if params[:simple]
-
-          present paginate(projects), options
+      def present_projects(options = {})
+        projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
+        projects = projects.with_statistics if params[:statistics]
+        projects = projects.with_issues_enabled if params[:with_issues_enabled]
+        projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+
+        if current_user
+          projects = projects.includes(:route, :taggings, namespace: :route)
+          project_members = current_user.project_members
+          group_members = current_user.group_members
         end
+
+        options = options.reverse_merge(
+          with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
+          statistics: params[:statistics],
+          project_members: project_members,
+          group_members: group_members,
+          current_user: current_user
+        )
+        options[:with] = Entities::BasicProjectDetails if params[:simple]
+
+        present paginate(projects), options
       end
+    end
 
+    resource :users, requirements: { user_id: %r{[^/]+} } do
+      desc 'Get a user projects' do
+        success Entities::BasicProjectDetails
+      end
+      params do
+        requires :user_id, type: String, desc: 'The ID or username of the user'
+        use :collection_params
+        use :statistics_params
+      end
+      get ":user_id/projects" do
+        user = find_user(params[:user_id])
+        not_found!('User') unless user
+
+        params[:user] = user
+
+        present_projects
+      end
+    end
+
+    resource :projects do
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
diff --git a/lib/api/scope.rb b/lib/api/scope.rb
new file mode 100644
index 00000000000..d5165b2e482
--- /dev/null
+++ b/lib/api/scope.rb
@@ -0,0 +1,23 @@
+# Encapsulate a scope used for authorization, such as `api`, or `read_user`
+module API
+  class Scope
+    attr_reader :name, :if
+
+    def initialize(name, options = {})
+      @name = name.to_sym
+      @if = options[:if]
+    end
+
+    # Are the `scopes` passed in sufficient to adequately authorize the passed
+    # request for the scope represented by the current instance of this class?
+    def sufficient?(scopes, request)
+      scopes.include?(self.name) && verify_if_condition(request)
+    end
+
+    private
+
+    def verify_if_condition(request)
+      self.if.nil? || self.if.call(request)
+    end
+  end
+end
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index d598f9a62a2..b19095d1252 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -65,6 +65,7 @@ module API
         :shared_runners_enabled,
         :sidekiq_throttling_enabled,
         :sign_in_text,
+        :password_authentication_enabled,
         :signin_enabled,
         :signup_enabled,
         :terminal_max_session_time,
@@ -95,7 +96,9 @@ module API
         requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
       end
       optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
-      optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled'
+      optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled'
+      optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled'
+      mutually_exclusive :password_authentication_enabled, :signin_enabled
       optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication'
       given require_two_factor_authentication: ->(val) { val } do
         requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
@@ -176,6 +179,10 @@ module API
     put "application/settings" do
       attrs = declared_params(include_missing: false)
 
+      if attrs.has_key?(:signin_enabled)
+        attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled)
+      end
+
       if current_settings.update_attributes(attrs)
         present current_settings, with: Entities::ApplicationSetting
       else
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index c630c24c339..fd634037a77 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -140,6 +140,22 @@ module API
         content_type 'text/plain'
         present snippet.content
       end
+
+      desc 'Get the user agent details for a snippet' do
+        success Entities::UserAgentDetail
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of a snippet'
+      end
+      get ":id/user_agent_detail" do
+        authenticated_as_admin!
+
+        snippet = Snippet.find_by!(id: params[:id])
+
+        return not_found!('UserAgentDetail') unless snippet.user_agent_detail
+
+        present snippet.user_agent_detail, with: Entities::UserAgentDetail
+      end
     end
   end
 end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 5b9d9a71be4..81c68ea2658 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -1,10 +1,9 @@
 module API
   class Users < Grape::API
     include PaginationParams
+    include APIGuard
 
-    before do
-      allow_access_with_scope :read_user if request.get?
-    end
+    allow_access_with_scope :read_user, if: -> (request) { request.get? }
 
     resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do
       before do
@@ -49,6 +48,8 @@ module API
         optional :active, type: Boolean, default: false, desc: 'Filters only active users'
         optional :external, type: Boolean, default: false, desc: 'Filters only external users'
         optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users'
+        optional :created_after, type: DateTime, desc: 'Return users created after the specified time'
+        optional :created_before, type: DateTime, desc: 'Return users created before the specified time'
         all_or_none_of :extern_uid, :provider
 
         use :pagination
@@ -56,6 +57,10 @@ module API
       get do
         authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?)
 
+        unless current_user&.admin?
+          params.except!(:created_after, :created_before)
+        end
+
         users = UsersFinder.new(current_user, params).execute
 
         authorized = can?(current_user, :read_users_list)
@@ -416,7 +421,16 @@ module API
         success Entities::UserPublic
       end
       get do
-        present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic
+        entity =
+          if sudo?
+            Entities::UserWithPrivateDetails
+          elsif current_user.admin?
+            Entities::UserWithAdmin
+          else
+            Entities::UserPublic
+          end
+
+        present current_user, with: entity
       end
 
       desc "Get the currently authenticated user's SSH keys" do
diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb
index c848f52723b..3759250f7f6 100644
--- a/lib/api/v3/entities.rb
+++ b/lib/api/v3/entities.rb
@@ -161,7 +161,8 @@ module API
         expose :id
         expose :default_projects_limit
         expose :signup_enabled
-        expose :signin_enabled
+        expose :password_authentication_enabled
+        expose :password_authentication_enabled, as: :signin_enabled
         expose :gravatar_enabled
         expose :sign_in_text
         expose :after_sign_up_text
diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb
index 748d6b97d4f..202011cfcbe 100644
--- a/lib/api/v3/settings.rb
+++ b/lib/api/v3/settings.rb
@@ -44,7 +44,9 @@ module API
           requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
         end
         optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
-        optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled'
+        optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled'
+        optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled'
+        mutually_exclusive :password_authentication_enabled, :signin_enabled
         optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication'
         given require_two_factor_authentication: ->(val) { val } do
           requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
@@ -116,7 +118,7 @@ module API
                         :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources,
                         :user_oauth_applications, :user_default_external, :signup_enabled,
                         :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled,
-                        :after_sign_up_text, :signin_enabled, :require_two_factor_authentication,
+                        :after_sign_up_text, :password_authentication_enabled, :signin_enabled, :require_two_factor_authentication,
                         :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text,
                         :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay,
                         :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled,
@@ -126,7 +128,13 @@ module API
                         :housekeeping_enabled, :terminal_max_session_time
       end
       put "application/settings" do
-        if current_settings.update_attributes(declared_params(include_missing: false))
+        attrs = declared_params(include_missing: false)
+
+        if attrs.has_key?(:signin_enabled)
+          attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled)
+        end
+
+        if current_settings.update_attributes(attrs)
           present current_settings, with: Entities::ApplicationSetting
         else
           render_validation_error!(current_settings)
diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb
index 37020019e07..cf106f2552d 100644
--- a/lib/api/v3/users.rb
+++ b/lib/api/v3/users.rb
@@ -2,9 +2,11 @@ module API
   module V3
     class Users < Grape::API
       include PaginationParams
+      include APIGuard
+
+      allow_access_with_scope :read_user, if: -> (request) { request.get? }
 
       before do
-        allow_access_with_scope :read_user if request.get?
         authenticate!
       end
 
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index 10374995497..7fa528fb2d3 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -45,7 +45,9 @@ module API
         optional :protected, type: String, desc: 'Whether the variable is protected'
       end
       post ':id/variables' do
-        variable = user_project.variables.create(declared_params(include_missing: false))
+        variable_params = declared_params(include_missing: false)
+
+        variable = user_project.variables.create(variable_params)
 
         if variable.valid?
           present variable, with: Entities::Variable
@@ -67,7 +69,9 @@ module API
 
         return not_found!('Variable') unless variable
 
-        if variable.update(declared_params(include_missing: false).except(:key))
+        variable_params = declared_params(include_missing: false).except(:key)
+
+        if variable.update(variable_params)
           present variable, with: Entities::Variable
         else
           render_validation_error!(variable)
diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb
index eaacb9591b1..21bcb1c5ca8 100644
--- a/lib/banzai/filter/commit_range_reference_filter.rb
+++ b/lib/banzai/filter/commit_range_reference_filter.rb
@@ -30,7 +30,7 @@ module Banzai
 
       def url_for_object(range, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_compare_url(project.namespace, project,
+        h.project_compare_url(project,
                                         range.to_param.merge(only_path: context[:only_path]))
       end
 
diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb
index 69c06117eda..714e0319025 100644
--- a/lib/banzai/filter/commit_reference_filter.rb
+++ b/lib/banzai/filter/commit_reference_filter.rb
@@ -24,7 +24,7 @@ module Banzai
 
       def url_for_object(commit, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_commit_url(project.namespace, project, commit,
+        h.project_commit_url(project, commit,
                                         only_path: context[:only_path])
       end
 
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index a605dea149e..5364984c9d3 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -61,8 +61,7 @@ module Banzai
 
       def url_for_object(label, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_issues_url(project.namespace, project, label_name: label.name,
-                                                                   only_path:  context[:only_path])
+        h.project_issues_url(project, label_name: label.name, only_path: context[:only_path])
       end
 
       def object_link_text(object, matches)
diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb
index 3888acf935e..0eab865ac04 100644
--- a/lib/banzai/filter/merge_request_reference_filter.rb
+++ b/lib/banzai/filter/merge_request_reference_filter.rb
@@ -17,7 +17,7 @@ module Banzai
 
       def url_for_object(mr, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_merge_request_url(project.namespace, project, mr,
+        h.project_merge_request_url(project, mr,
                                             only_path: context[:only_path])
       end
 
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index f12014e191f..45c033d32a8 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -49,7 +49,7 @@ module Banzai
 
       def url_for_object(milestone, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_milestone_url(project.namespace, project, milestone,
+        h.project_milestone_url(project, milestone,
                                         only_path: context[:only_path])
       end
 
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 6640168bfa2..a6f8650ed3d 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -30,6 +30,8 @@ module Banzai
         attributes = attributes.reject { |_, v| v.nil? }
 
         attributes[:reference_type] ||= self.class.reference_type
+        attributes[:container] ||= 'body'
+        attributes[:placement] ||= 'bottom'
         attributes.delete(:original) if context[:no_original_data]
         attributes.map do |key, value|
           %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}")
diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb
index 212a0bbf2a0..134a192c22b 100644
--- a/lib/banzai/filter/snippet_reference_filter.rb
+++ b/lib/banzai/filter/snippet_reference_filter.rb
@@ -17,7 +17,7 @@ module Banzai
 
       def url_for_object(snippet, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_snippet_url(project.namespace, project, snippet,
+        h.project_snippet_url(project, snippet,
                                         only_path: context[:only_path])
       end
     end
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index a798927823f..f3356d6c51e 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -107,7 +107,7 @@ module Banzai
         if author && !project.team.member?(author)
           link_content
         else
-          url = urls.namespace_project_url(project.namespace, project,
+          url = urls.project_url(project,
                                            only_path: context[:only_path])
 
           data = data_attribute(project: project.id, author: author.try(:id))
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 56ad2c77c7d..cf3a0336792 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -80,6 +80,8 @@ module Ci
           artifacts: job[:artifacts],
           cache: job[:cache],
           dependencies: job[:dependencies],
+          before_script: job[:before_script],
+          script: job[:script],
           after_script: job[:after_script],
           environment: job[:environment]
         }.compact }
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index dd864eea3fa..721ed97bb6b 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -126,8 +126,7 @@ module ExtractsPath
     raise InvalidPathError unless @commit
 
     @hex_path = Digest::SHA1.hexdigest(@path)
-    @logs_path = logs_file_namespace_project_ref_path(@project.namespace,
-                                                      @project, @ref, @path)
+    @logs_path = logs_file_project_ref_path(@project, @ref, @path)
 
   rescue RuntimeError, NoMethodError, InvalidPathError
     render_404
diff --git a/lib/feature.rb b/lib/feature.rb
index 363f66ba60e..4bd29aed687 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -57,5 +57,11 @@ class Feature
         Flipper.new(adapter)
       end
     end
+
+    # This method is called from config/initializers/flipper.rb and can be used
+    # to register Flipper groups.
+    # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups
+    def register_feature_groups
+    end
   end
 end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 3933c3b04dd..9bed81e7327 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -37,7 +37,7 @@ module Gitlab
         rate_limit!(ip, success: result.success?, login: login)
         Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor)
 
-        return result if result.success? || current_application_settings.signin_enabled? || Gitlab::LDAP::Config.enabled?
+        return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled?
 
         # If sign-in is disabled and LDAP is not configured, recommend a
         # personal access token on failed auth attempts
@@ -48,6 +48,10 @@ module Gitlab
         # Avoid resource intensive login checks if password is not provided
         return unless password.present?
 
+        # Nothing to do here if internal auth is disabled and LDAP is
+        # not configured
+        return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled?
+
         Gitlab::Auth::UniqueIpsLimiter.limit_user! do
           user = User.by_login(login)
 
@@ -130,13 +134,13 @@ module Gitlab
 
         token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
 
-        if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s))
+        if token && valid_scoped_token?(token, AVAILABLE_SCOPES)
           Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes))
         end
       end
 
       def valid_oauth_token?(token)
-        token && token.accessible? && valid_scoped_token?(token, ["api"])
+        token && token.accessible? && valid_scoped_token?(token, [:api])
       end
 
       def valid_scoped_token?(token, scopes)
diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb
index bf2239ca150..baa1f802d8a 100644
--- a/lib/gitlab/auth/unique_ips_limiter.rb
+++ b/lib/gitlab/auth/unique_ips_limiter.rb
@@ -27,7 +27,7 @@ module Gitlab
           time = Time.now.utc.to_i
           key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}"
 
-          Gitlab::Redis.with do |redis|
+          Gitlab::Redis::SharedState.with do |redis|
             unique_ips_count = nil
             redis.multi do |r|
               r.zadd(key, time, ip)
diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb
index d95ecd7b291..b0741b1fba7 100644
--- a/lib/gitlab/background_migration.rb
+++ b/lib/gitlab/background_migration.rb
@@ -1,24 +1,45 @@
 module Gitlab
   module BackgroundMigration
+    def self.queue
+      @queue ||= BackgroundMigrationWorker.sidekiq_options['queue']
+    end
+
     # Begins stealing jobs from the background migrations queue, blocking the
     # caller until all jobs have been completed.
     #
+    # When a migration raises a StandardError is is going to be retries up to
+    # three times, for example, to recover from a deadlock.
+    #
+    # When Exception is being raised, it enqueues the migration again, and
+    # re-raises the exception.
+    #
     # steal_class - The name of the class for which to steal jobs.
     def self.steal(steal_class)
-      queue = Sidekiq::Queue
-        .new(BackgroundMigrationWorker.sidekiq_options['queue'])
+      enqueued = Sidekiq::Queue.new(self.queue)
+      scheduled = Sidekiq::ScheduledSet.new
 
-      queue.each do |job|
-        migration_class, migration_args = job.args
+      [scheduled, enqueued].each do |queue|
+        queue.each do |job|
+          migration_class, migration_args = job.args
 
-        next unless migration_class == steal_class
+          next unless job.queue == self.queue
+          next unless migration_class == steal_class
 
-        perform(migration_class, migration_args)
+          begin
+            perform(migration_class, migration_args, retries: 3) if job.delete
+          rescue Exception # rubocop:disable Lint/RescueException
+            BackgroundMigrationWorker # enqueue this migration again
+              .perform_async(migration_class, migration_args)
 
-        job.delete
+            raise
+          end
+        end
       end
     end
 
+    ##
+    # Performs a background migration.
+    #
     # class_name - The name of the background migration class as defined in the
     #              Gitlab::BackgroundMigration namespace.
     #
diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
new file mode 100644
index 00000000000..91540127ea9
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module BackgroundMigration
+    class MigrateBuildStageIdReference
+      def perform(start_id, stop_id)
+        sql = <<-SQL.strip_heredoc
+          UPDATE ci_builds
+            SET stage_id =
+              (SELECT id FROM ci_stages
+                WHERE ci_stages.pipeline_id = ci_builds.commit_id
+                AND ci_stages.name = ci_builds.stage)
+          WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+            AND ci_builds.stage_id IS NULL
+        SQL
+
+        ActiveRecord::Base.connection.execute(sql)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb
index f87a7b7942e..2ee35a0d4c1 100644
--- a/lib/gitlab/badge/build/metadata.rb
+++ b/lib/gitlab/badge/build/metadata.rb
@@ -15,12 +15,11 @@ module Gitlab
         end
 
         def image_url
-          build_namespace_project_badges_url(@project.namespace,
-                                             @project, @ref, format: :svg)
+          build_project_badges_url(@project, @ref, format: :svg)
         end
 
         def link_url
-          namespace_project_commits_url(@project.namespace, @project, id: @ref)
+          project_commits_url(@project, id: @ref)
         end
       end
     end
diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb
index 53588185622..e898f5d790e 100644
--- a/lib/gitlab/badge/coverage/metadata.rb
+++ b/lib/gitlab/badge/coverage/metadata.rb
@@ -16,13 +16,11 @@ module Gitlab
         end
 
         def image_url
-          coverage_namespace_project_badges_url(@project.namespace,
-                                                @project, @ref,
-                                                format: :svg)
+          coverage_project_badges_url(@project, @ref, format: :svg)
         end
 
         def link_url
-          namespace_project_commits_url(@project.namespace, @project, id: @ref)
+          project_commits_url(@project, @ref)
         end
       end
     end
diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb
index 4a049ef758d..8ad6f3cb986 100644
--- a/lib/gitlab/badge/metadata.rb
+++ b/lib/gitlab/badge/metadata.rb
@@ -4,7 +4,7 @@ module Gitlab
     # Abstract class for badge metadata
     #
     class Metadata
-      include Gitlab::Application.routes.url_helpers
+      include Gitlab::Routing
       include ActionView::Helpers::AssetTagHelper
       include ActionView::Helpers::UrlHelper
 
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
index 9c2e09943b0..dba37892863 100644
--- a/lib/gitlab/cache/ci/project_pipeline_status.rb
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -23,7 +23,7 @@ module Gitlab
         end
 
         def self.cached_results_for_projects(projects)
-          result = Gitlab::Redis.with do |redis|
+          result = Gitlab::Redis::Cache.with do |redis|
             redis.multi do
               projects.each do |project|
                 cache_key = cache_key_for_project(project)
@@ -100,19 +100,19 @@ module Gitlab
         end
 
         def load_from_cache
-          Gitlab::Redis.with do |redis|
+          Gitlab::Redis::Cache.with do |redis|
             self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref)
           end
         end
 
         def store_in_cache
-          Gitlab::Redis.with do |redis|
+          Gitlab::Redis::Cache.with do |redis|
             redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
           end
         end
 
         def delete_from_cache
-          Gitlab::Redis.with do |redis|
+          Gitlab::Redis::Cache.with do |redis|
             redis.del(cache_key)
           end
         end
@@ -120,7 +120,7 @@ module Gitlab
         def has_cache?
           return self.loaded unless self.loaded.nil?
 
-          Gitlab::Redis.with do |redis|
+          Gitlab::Redis::Cache.with do |redis|
             redis.exists(cache_key)
           end
         end
diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb
index 1b081aa9b1d..e63e5437331 100644
--- a/lib/gitlab/chat_name_token.rb
+++ b/lib/gitlab/chat_name_token.rb
@@ -12,23 +12,23 @@ module Gitlab
     end
 
     def get
-      Gitlab::Redis.with do |redis|
-        data = redis.get(redis_key)
+      Gitlab::Redis::SharedState.with do |redis|
+        data = redis.get(redis_shared_state_key)
         JSON.parse(data, symbolize_names: true) if data
       end
     end
 
     def store!(params)
-      Gitlab::Redis.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
         params = params.to_json
-        redis.set(redis_key, params, ex: EXPIRY_TIME)
+        redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME)
         token
       end
     end
 
     def delete
-      Gitlab::Redis.with do |redis|
-        redis.del(redis_key)
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.del(redis_shared_state_key)
       end
     end
 
@@ -38,7 +38,7 @@ module Gitlab
       Devise.friendly_token(TOKEN_LENGTH)
     end
 
-    def redis_key
+    def redis_shared_state_key
       "gitlab:chat_names:#{token}"
     end
   end
diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
index ee034d9cc56..411f67f8ce7 100644
--- a/lib/gitlab/ci/build/step.rb
+++ b/lib/gitlab/ci/build/step.rb
@@ -12,7 +12,8 @@ module Gitlab
         class << self
           def from_commands(job)
             self.new(:script).tap do |step|
-              step.script = job.commands.split("\n")
+              step.script = job.options[:before_script].to_a + job.options[:script].to_a
+              step.script = job.commands.split("\n") if step.script.empty?
               step.timeout = job.timeout
               step.when = WHEN_ON_SUCCESS
             end
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index f074df9c7a1..d7e09acbbf3 100644
--- a/lib/gitlab/ci/config/entry/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -7,11 +7,14 @@ module Gitlab
         #
         class Cache < Node
           include Configurable
+          include Attributable
 
-          ALLOWED_KEYS = %i[key untracked paths].freeze
+          ALLOWED_KEYS = %i[key untracked paths policy].freeze
+          DEFAULT_POLICY = 'pull-push'.freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
+            validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true
           end
 
           entry :key, Entry::Key,
@@ -25,8 +28,15 @@ module Gitlab
 
           helpers :key
 
+          attributes :policy
+
           def value
-            super.merge(key: key_value)
+            result = super
+
+            result[:key] = key_value
+            result[:policy] = policy || DEFAULT_POLICY
+
+            result
           end
         end
       end
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
index 439ef0ce015..8ad3e57e59d 100644
--- a/lib/gitlab/ci/status/build/cancelable.rb
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -12,9 +12,7 @@ module Gitlab
           end
 
           def action_path
-            cancel_namespace_project_job_path(subject.project.namespace,
-                                                subject.project,
-                                                subject)
+            cancel_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
index b173c23fba4..c0c7c7f5b5d 100644
--- a/lib/gitlab/ci/status/build/common.rb
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -8,9 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_job_path(subject.project.namespace,
-                                         subject.project,
-                                         subject)
+            project_job_path(subject.project, subject)
           end
         end
       end
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
index e80f3263794..c7726543599 100644
--- a/lib/gitlab/ci/status/build/play.rb
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -20,9 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_job_path(subject.project.namespace,
-                                              subject.project,
-                                              subject)
+            play_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
index 56303e4cb17..8c8fdc56d75 100644
--- a/lib/gitlab/ci/status/build/retryable.rb
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -16,9 +16,7 @@ module Gitlab
           end
 
           def action_path
-            retry_namespace_project_job_path(subject.project.namespace,
-                                               subject.project,
-                                               subject)
+            retry_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
index 2778d6f3b52..d464738deaf 100644
--- a/lib/gitlab/ci/status/build/stop.rb
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -20,9 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_job_path(subject.project.namespace,
-                                              subject.project,
-                                              subject)
+            play_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb
index 76bfd18bf40..61bb07beb0f 100644
--- a/lib/gitlab/ci/status/pipeline/common.rb
+++ b/lib/gitlab/ci/status/pipeline/common.rb
@@ -8,9 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_pipeline_path(subject.project.namespace,
-                                            subject.project,
-                                            subject)
+            project_pipeline_path(subject.project, subject)
           end
 
           def has_action?
diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb
index 7852f492e1d..bc99d925347 100644
--- a/lib/gitlab/ci/status/stage/common.rb
+++ b/lib/gitlab/ci/status/stage/common.rb
@@ -8,10 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_pipeline_path(subject.project.namespace,
-                                            subject.project,
-                                            subject.pipeline,
-                                            anchor: subject.name)
+            project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name)
           end
 
           def has_action?
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
index 75a213ef752..98dfe900044 100644
--- a/lib/gitlab/conflict/file.rb
+++ b/lib/gitlab/conflict/file.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Conflict
     class File
-      include Gitlab::Routing.url_helpers
+      include Gitlab::Routing
       include IconsHelper
 
       MissingResolution = Class.new(ResolutionError)
@@ -205,9 +205,7 @@ module Gitlab
           old_path: their_path,
           new_path: our_path,
           blob_icon: file_type_icon_class('file', our_mode, our_path),
-          blob_path: namespace_project_blob_path(merge_request.project.namespace,
-                                                 merge_request.project,
-                                                 ::File.join(merge_request.diff_refs.head_sha, our_path))
+          blob_path: project_blob_path(merge_request.project, ::File.join(merge_request.diff_refs.head_sha, our_path))
         }
 
         json_hash.tap do |json_hash|
@@ -223,11 +221,10 @@ module Gitlab
       end
 
       def content_path
-        conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace,
-                                                               merge_request.project,
-                                                               merge_request,
-                                                               old_path: their_path,
-                                                               new_path: our_path)
+        conflict_for_path_project_merge_request_path(merge_request.project,
+                                                     merge_request,
+                                                     old_path: their_path,
+                                                     new_path: our_path)
       end
 
       # Don't try to print merge_request or repository.
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 818b3d9c46b..7fa02f3d7b3 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -25,7 +25,7 @@ module Gitlab
     def cached_application_settings
       begin
         ::ApplicationSetting.cached
-      rescue ::Redis::BaseError, ::Errno::ENOENT
+      rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL
         # In case Redis isn't running or the Redis UNIX socket file is not available
       end
     end
@@ -33,12 +33,7 @@ module Gitlab
     def uncached_application_settings
       return fake_application_settings unless connect_to_db?
 
-      # This loads from the database into the cache, so handle Redis errors
-      begin
-        db_settings = ::ApplicationSetting.current
-      rescue ::Redis::BaseError, ::Errno::ENOENT
-        # In case Redis isn't running or the Redis UNIX socket file is not available
-      end
+      db_settings = ::ApplicationSetting.current
 
       # If there are pending migrations, it's possible there are columns that
       # need to be added to the application settings. To prevent Rake tasks
diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb
index 9d25ef078e8..f5d08c0b658 100644
--- a/lib/gitlab/cycle_analytics/metrics_tables.rb
+++ b/lib/gitlab/cycle_analytics/metrics_tables.rb
@@ -13,6 +13,10 @@ module Gitlab
         MergeRequestDiff.arel_table
       end
 
+      def mr_diff_commits_table
+        MergeRequestDiffCommit.arel_table
+      end
+
       def mr_closing_issues_table
         MergeRequestsClosingIssues.arel_table
       end
diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
index 7d342a2d2cb..b260822788d 100644
--- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
@@ -2,40 +2,59 @@ module Gitlab
   module CycleAnalytics
     class PlanEventFetcher < BaseEventFetcher
       def initialize(*args)
-        @projections = [mr_diff_table[:st_commits].as('commits'),
+        @projections = [mr_diff_table[:id],
+                        mr_diff_table[:st_commits],
                         issue_metrics_table[:first_mentioned_in_commit_at]]
 
         super(*args)
       end
 
       def events_query
-        base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id]))
+        base_query
+          .join(mr_diff_table)
+          .on(mr_diff_table[:merge_request_id].eq(mr_table[:id]))
 
         super
       end
 
       private
 
+      def merge_request_diff_commits
+        @merge_request_diff_commits ||=
+          MergeRequestDiffCommit
+            .where(merge_request_diff_id: event_result.map { |event| event['id'] })
+            .group_by(&:merge_request_diff_id)
+      end
+
       def serialize(event)
-        st_commit = first_time_reference_commit(event.delete('commits'), event)
+        commit = first_time_reference_commit(event)
 
-        return unless st_commit
+        return unless commit
 
-        serialize_commit(event, st_commit, query)
+        serialize_commit(event, commit, query)
       end
 
-      def first_time_reference_commit(commits, event)
+      def first_time_reference_commit(event)
+        return nil unless event && merge_request_diff_commits
+
+        commits =
+          if event['st_commits'].present?
+            YAML.load(event['st_commits'])
+          else
+            merge_request_diff_commits[event['id'].to_i]
+          end
+
         return nil if commits.blank?
 
-        YAML.load(commits).find do |commit|
+        commits.find do |commit|
           next unless commit[:committed_date] && event['first_mentioned_in_commit_at']
 
           commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i
         end
       end
 
-      def serialize_commit(event, st_commit, query)
-        commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project)
+      def serialize_commit(event, commit, query)
+        commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project)
 
         AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit)
       end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index 33f8939bc61..1a697396ff1 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -145,7 +145,7 @@ module Gitlab
 
           def track_rename(type, old_path, new_path)
             key = redis_key_for_type(type)
-            Gitlab::Redis.with do |redis|
+            Gitlab::Redis::SharedState.with do |redis|
               redis.lpush(key, [old_path, new_path].to_json)
               redis.expire(key, 2.weeks.to_i)
             end
@@ -155,7 +155,7 @@ module Gitlab
           def reverts_for_type(type)
             key = redis_key_for_type(type)
 
-            Gitlab::Redis.with do |redis|
+            Gitlab::Redis::SharedState.with do |redis|
               failed_reverts = []
 
               while rename_info = redis.lpop(key)
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index 1a5887dab7e..edd7795eef0 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -337,7 +337,7 @@ module Gitlab
           # In the EE repo
           $ git push origin #{ee_branch_prefix}
 
-        ⚠️ Also, don't forget to create a new merge request on gitlab-ce and
+        ⚠️ Also, don't forget to create a new merge request on gitlab-ee and
         cross-link it with the CE merge request.
 
         Once this is done, you can retry this failed build, and it should pass.
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index ea035e33eff..dd1d9dcd555 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -4,7 +4,7 @@ module Gitlab
       class RepositoryPush
         attr_reader :author_id, :ref, :action
 
-        include Gitlab::Routing.url_helpers
+        include Gitlab::Routing
         include DiffHelper
 
         delegate :namespace, :name_with_namespace, to: :project, prefix: :project
@@ -96,20 +96,13 @@ module Gitlab
         def target_url
           if @action == :push && commits
             if commits.length > 1
-              namespace_project_compare_url(project_namespace,
-                                            project,
-                                            from: compare.start_commit,
-                                            to:   compare.head_commit)
+              project_compare_url(project, from: compare.start_commit, to: compare.head_commit)
             else
-              namespace_project_commit_url(project_namespace,
-                                           project,
-                                           commits.first)
+              project_commit_url(project, commits.first)
             end
           else
             unless @action == :delete
-              namespace_project_tree_url(project_namespace,
-                                         project,
-                                         ref_name)
+              project_tree_url(project, ref_name)
             end
           end
         end
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
index 072fcfc65e6..21172ff8d93 100644
--- a/lib/gitlab/etag_caching/store.rb
+++ b/lib/gitlab/etag_caching/store.rb
@@ -2,17 +2,17 @@ module Gitlab
   module EtagCaching
     class Store
       EXPIRY_TIME = 20.minutes
-      REDIS_NAMESPACE = 'etag:'.freeze
+      SHARED_STATE_NAMESPACE = 'etag:'.freeze
 
       def get(key)
-        Gitlab::Redis.with { |redis| redis.get(redis_key(key)) }
+        Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) }
       end
 
       def touch(key, only_if_missing: false)
         etag = generate_etag
 
-        Gitlab::Redis.with do |redis|
-          redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
+        Gitlab::Redis::SharedState.with do |redis|
+          redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
         end
 
         etag
@@ -24,10 +24,10 @@ module Gitlab
         SecureRandom.hex
       end
 
-      def redis_key(key)
+      def redis_shared_state_key(key)
         raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key)
 
-        "#{REDIS_NAMESPACE}#{key}"
+        "#{SHARED_STATE_NAMESPACE}#{key}"
       end
     end
   end
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index a0f46594eb1..3784f6c4947 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -26,17 +26,17 @@ module Gitlab
     EOS
 
     def self.cancel(key, uuid)
-      Gitlab::Redis.with do |redis|
-        redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid])
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid])
       end
     end
 
-    def self.redis_key(key)
+    def self.redis_shared_state_key(key)
       "gitlab:exclusive_lease:#{key}"
     end
 
     def initialize(key, timeout:)
-      @redis_key = self.class.redis_key(key)
+      @redis_shared_state_key = self.class.redis_shared_state_key(key)
       @timeout = timeout
       @uuid = SecureRandom.uuid
     end
@@ -45,24 +45,24 @@ module Gitlab
     # false if the lease is already taken.
     def try_obtain
       # Performing a single SET is atomic
-      Gitlab::Redis.with do |redis|
-        redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid
       end
     end
 
     # Try to renew an existing lease. Return lease UUID on success,
     # false if the lease is taken by a different UUID or inexistent.
     def renew
-      Gitlab::Redis.with do |redis|
-        result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout])
+      Gitlab::Redis::SharedState.with do |redis|
+        result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout])
         result == @uuid
       end
     end
 
     # Returns true if the key for this lease is set.
     def exists?
-      Gitlab::Redis.with do |redis|
-        redis.exists(@redis_key)
+      Gitlab::Redis::SharedState.with do |redis|
+        redis.exists(@redis_shared_state_key)
       end
     end
   end
diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb
index 42140ecc993..2d20cd473a7 100644
--- a/lib/gitlab/git/attributes.rb
+++ b/lib/gitlab/git/attributes.rb
@@ -1,3 +1,8 @@
+# Gitaly note: JV: not sure what to make of this class. Why does it use
+# the full disk path of the repository to look up attributes This is
+# problematic in Gitaly, because Gitaly hides the full disk path to the
+# repository from gitlab-ce.
+
 module Gitlab
   module Git
     # Class for parsing Git attribute files and extracting the attributes for
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 66829a03c2e..0deaab01b5b 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: needs 1 RPC for #load_blame.
+
 module Gitlab
   module Git
     class Blame
@@ -24,6 +26,7 @@ module Gitlab
 
       private
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376
       def load_blame
         cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
         # Read in binary mode to ensure ASCII-8BIT
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index a7aceab4c14..b6dd3cd20e0 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: seems to be completely migrated (behind feature flags).
+
 module Gitlab
   module Git
     class Blob
@@ -41,10 +43,6 @@ module Gitlab
               commit_id: sha
             )
           when :BLOB
-            # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
-            # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
-            # which is what we use below to keep a consistent behavior.
-            detect = CharlockHolmes::EncodingDetector.new(8000).detect(entry.data)
             new(
               id: entry.oid,
               name: name,
@@ -53,7 +51,7 @@ module Gitlab
               mode: entry.mode.to_s(8),
               path: path,
               commit_id: sha,
-              binary: detect && detect[:type] == :binary
+              binary: binary?(entry.data)
             )
           end
         end
@@ -87,16 +85,32 @@ module Gitlab
         end
 
         def raw(repository, sha)
-          blob = repository.lookup(sha)
+          Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
+            if is_enabled
+              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+            else
+              blob = repository.lookup(sha)
+    
+              new(
+                id: blob.oid,
+                size: blob.size,
+                data: blob.content(MAX_DATA_DISPLAY_SIZE),
+                binary: blob.binary?
+              )
+            end
+          end
+        end
 
-          new(
-            id: blob.oid,
-            size: blob.size,
-            data: blob.content(MAX_DATA_DISPLAY_SIZE),
-            binary: blob.binary?
-          )
+        def binary?(data)
+          # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
+          # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
+          # which is what we use below to keep a consistent behavior.
+          detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
+          detect && detect[:type] == :binary
         end
 
+        private
+
         # Recursive search of blob id by path
         #
         # Ex.
@@ -165,8 +179,17 @@ module Gitlab
         return if @data == '' # don't mess with submodule blobs
         return @data if @loaded_all_data
 
+        Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
+          @data = begin
+            if is_enabled
+              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data
+            else
+              repository.lookup(id).content
+            end
+          end
+        end
+
         @loaded_all_data = true
-        @data = repository.lookup(id).content
         @loaded_size = @data.bytesize
         @binary = nil
       end
@@ -175,6 +198,10 @@ module Gitlab
         encode! @name
       end
 
+      def path
+        encode! @path
+      end
+
       def truncated?
         size && (size > loaded_size)
       end
diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb
index d7975f88aaa..68116e775c6 100644
--- a/lib/gitlab/git/blob_snippet.rb
+++ b/lib/gitlab/git/blob_snippet.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class BlobSnippet
diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb
index 124526e4b59..c53882787f1 100644
--- a/lib/gitlab/git/branch.rb
+++ b/lib/gitlab/git/branch.rb
@@ -1,39 +1,10 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class Branch < Ref
-      def initialize(repository, name, target)
-        if target.is_a?(Gitaly::FindLocalBranchResponse)
-          target = target_from_gitaly_local_branches_response(target)
-        end
-
-        super(repository, name, target)
-      end
-
-      def target_from_gitaly_local_branches_response(response)
-        # Git messages have no encoding enforcements. However, in the UI we only
-        # handle UTF-8, so basically we cross our fingers that the message force
-        # encoded to UTF-8 is readable.
-        message = response.commit_subject.dup.force_encoding('UTF-8')
-
-        # NOTE: For ease of parsing in Gitaly, we have only the subject of
-        # the commit and not the full message. This is ok, since all the
-        # code that uses `local_branches` only cares at most about the
-        # commit message.
-        # TODO: Once gitaly "takes over" Rugged consider separating the
-        # subject from the message to make it clearer when there's one
-        # available but not the other.
-        hash = {
-          id: response.commit_id,
-          message: message,
-          authored_date: Time.at(response.commit_author.date.seconds),
-          author_name: response.commit_author.name,
-          author_email: response.commit_author.email,
-          committed_date: Time.at(response.commit_committer.date.seconds),
-          committer_name: response.commit_committer.name,
-          committer_email: response.commit_committer.email
-        }
-
-        Gitlab::Git::Commit.decorate(hash)
+      def initialize(repository, name, target, target_commit)
+        super(repository, name, target, target_commit)
       end
     end
   end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 9c0606d780a..d0f04d25db2 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -38,7 +38,7 @@ module Gitlab
           repo = options.delete(:repo)
           raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log)
 
-          repo.log(options).map { |c| decorate(c) }
+          repo.log(options)
         end
 
         # Get single commit
@@ -48,6 +48,7 @@ module Gitlab
         #
         #   Commit.find(repo, 'master')
         #
+        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321
         def find(repo, commit_id = "HEAD")
           return commit_id if commit_id.is_a?(Gitlab::Git::Commit)
           return decorate(commit_id) if commit_id.is_a?(Rugged::Commit)
@@ -124,6 +125,7 @@ module Gitlab
         #        are documented here:
         #        http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant)
         #
+        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326
         def find_all(repo, options = {})
           actual_options = options.dup
 
@@ -243,6 +245,8 @@ module Gitlab
       # Shows the diff between the commit's parent and the commit.
       #
       # Cuts out the header and stats from #to_patch and returns only the diff.
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324
       def to_diff
         diff_from_parent.patch
       end
diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb
index e9118bbed0e..57c29ad112c 100644
--- a/lib/gitlab/git/commit_stats.rb
+++ b/lib/gitlab/git/commit_stats.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: 1 RPC, migration in progress.
+
 # Gitlab::Git::CommitStats counts the additions, deletions, and total changes
 # in a commit.
 module Gitlab
@@ -6,6 +8,8 @@ module Gitlab
       attr_reader :id, :additions, :deletions, :total
 
       # Instantiate a CommitStats object
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323
       def initialize(commit)
         @id = commit.id
         @additions = 0
diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb
index 78e440395a5..7cb842256d0 100644
--- a/lib/gitlab/git/compare.rb
+++ b/lib/gitlab/git/compare.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class Compare
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index cf7829a583b..cf95f673667 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between.
+
 # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object
 module Gitlab
   module Git
@@ -81,110 +83,16 @@ module Gitlab
         # Return a copy of the +options+ hash containing only keys that can be
         # passed to Rugged.  Allowed options are:
         #
-        #  :max_size ::
-        #    An integer specifying the maximum byte size of a file before a it
-        #    will be treated as binary. The default value is 512MB.
-        #
-        #  :context_lines ::
-        #    The number of unchanged lines that define the boundary of a hunk
-        #    (and to display before and after the actual changes). The default is
-        #    3.
-        #
-        #  :interhunk_lines ::
-        #    The maximum number of unchanged lines between hunk boundaries before
-        #    the hunks will be merged into a one. The default is 0.
-        #
-        #  :old_prefix ::
-        #    The virtual "directory" to prefix to old filenames in hunk headers.
-        #    The default is "a".
-        #
-        #  :new_prefix ::
-        #    The virtual "directory" to prefix to new filenames in hunk headers.
-        #    The default is "b".
-        #
-        #  :reverse ::
-        #    If true, the sides of the diff will be reversed.
-        #
-        #  :force_text ::
-        #    If true, all files will be treated as text, disabling binary
-        #    attributes & detection.
-        #
-        #  :ignore_whitespace ::
-        #    If true, all whitespace will be ignored.
-        #
         #  :ignore_whitespace_change ::
         #    If true, changes in amount of whitespace will be ignored.
         #
-        #  :ignore_whitespace_eol ::
-        #    If true, whitespace at end of line will be ignored.
-        #
-        #  :ignore_submodules ::
-        #    if true, submodules will be excluded from the diff completely.
-        #
-        #  :patience ::
-        #    If true, the "patience diff" algorithm will be used (currenlty
-        #    unimplemented).
-        #
-        #  :include_ignored ::
-        #    If true, ignored files will be included in the diff.
-        #
-        #  :include_untracked ::
-        #   If true, untracked files will be included in the diff.
-        #
-        #  :include_unmodified ::
-        #    If true, unmodified files will be included in the diff.
-        #
-        #  :recurse_untracked_dirs ::
-        #    Even if +:include_untracked+ is true, untracked directories will
-        #    only be marked with a single entry in the diff. If this flag is set
-        #    to true, all files under ignored directories will be included in the
-        #    diff, too.
-        #
         #  :disable_pathspec_match ::
         #    If true, the given +*paths+ will be applied as exact matches,
         #    instead of as fnmatch patterns.
         #
-        #  :deltas_are_icase ::
-        #    If true, filename comparisons will be made with case-insensitivity.
-        #
-        #  :include_untracked_content ::
-        #    if true, untracked content will be contained in the the diff patch
-        #    text.
-        #
-        #  :skip_binary_check ::
-        #    If true, diff deltas will be generated without spending time on
-        #    binary detection. This is useful to improve performance in cases
-        #    where the actual file content difference is not needed.
-        #
-        #  :include_typechange ::
-        #    If true, type changes for files will not be interpreted as deletion
-        #    of the "old file" and addition of the "new file", but will generate
-        #    typechange records.
-        #
-        #  :include_typechange_trees ::
-        #    Even if +:include_typechange+ is true, blob -> tree changes will
-        #    still usually be handled as a deletion of the blob. If this flag is
-        #    set to true, blob -> tree changes will be marked as typechanges.
-        #
-        #  :ignore_filemode ::
-        #    If true, file mode changes will be ignored.
-        #
-        #  :recurse_ignored_dirs ::
-        #    Even if +:include_ignored+ is true, ignored directories will only be
-        #    marked with a single entry in the diff. If this flag is set to true,
-        #    all files under ignored directories will be included in the diff,
-        #    too.
         def filter_diff_options(options, default_options = {})
-          allowed_options = [:max_size, :context_lines, :interhunk_lines,
-                             :old_prefix, :new_prefix, :reverse, :force_text,
-                             :ignore_whitespace, :ignore_whitespace_change,
-                             :ignore_whitespace_eol, :ignore_submodules,
-                             :patience, :include_ignored, :include_untracked,
-                             :include_unmodified, :recurse_untracked_dirs,
-                             :disable_pathspec_match, :deltas_are_icase,
-                             :include_untracked_content, :skip_binary_check,
-                             :include_typechange, :include_typechange_trees,
-                             :ignore_filemode, :recurse_ignored_dirs, :paths,
+          allowed_options = [:ignore_whitespace_change,
+                             :disable_pathspec_match, :paths,
                              :max_files, :max_lines, :limits, :expanded]
 
           if default_options
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 555894907cc..0d8fe185ac5 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class DiffCollection
diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb
index 0fdc57ec954..f80193ac553 100644
--- a/lib/gitlab/git/env.rb
+++ b/lib/gitlab/git/env.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     # Ephemeral (per request) storage for environment variables that some Git
diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb
index f4e3b5e5129..4a43b9b444d 100644
--- a/lib/gitlab/git/gitmodules_parser.rb
+++ b/lib/gitlab/git/gitmodules_parser.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class GitmodulesParser
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index 5042916343b..8f0c377ef4f 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -1,3 +1,7 @@
+# Gitaly note: JV: looks like this is only used by GitHooksService in
+# app/services. We shouldn't bother migrating this until we know how
+# GitHooksService will be migrated.
+
 module Gitlab
   module Git
     class Hook
diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb
index 1add037fa5f..db532600d1b 100644
--- a/lib/gitlab/git/index.rb
+++ b/lib/gitlab/git/index.rb
@@ -1,3 +1,7 @@
+# Gitaly note: JV: When the time comes I think we will want to copy this
+# class into Gitaly. None of its methods look like they should be RPC's.
+# The RPC's will be at a higher level.
+
 module Gitlab
   module Git
     class Index
@@ -110,10 +114,6 @@ module Gitlab
           if segment == '..'
             raise IndexError, 'Path cannot include directory traversal'
           end
-
-          unless segment =~ Gitlab::Regex.file_name_regex
-            raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}"
-          end
         end
 
         pathname.to_s
diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb
index 0148cd8df05..42c80aabd0a 100644
--- a/lib/gitlab/git/path_helper.rb
+++ b/lib/gitlab/git/path_helper.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     class PathHelper
diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb
index df9ca3ee5ac..25fa62ce4bd 100644
--- a/lib/gitlab/git/popen.rb
+++ b/lib/gitlab/git/popen.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 require 'open3'
 
 module Gitlab
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index ebf7393dc61..372ce005b94 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged).
+
 module Gitlab
   module Git
     class Ref
@@ -24,16 +26,16 @@ module Gitlab
         str.gsub(/\Arefs\/heads\//, '')
       end
 
+      # Gitaly: this method will probably be migrated indirectly via its call sites.
       def self.dereference_object(object)
         object = object.target while object.is_a?(Rugged::Tag::Annotation)
 
         object
       end
 
-      def initialize(repository, name, target)
-        encode! name
-        @name = name.gsub(/\Arefs\/(tags|heads)\//, '')
-        @dereferenced_target = Gitlab::Git::Commit.find(repository, target)
+      def initialize(repository, name, target, derefenced_target)
+        @name = Gitlab::Git.ref_name(name)
+        @dereferenced_target = derefenced_target
         @target = if target.respond_to?(:oid)
                     target.oid
                   elsif target.respond_to?(:name)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 23d0c8a9bdb..5aefa1404ad 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -80,16 +80,10 @@ module Gitlab
       end
 
       # Returns an Array of Branches
-      def branches(filter: nil, sort_by: nil)
-        branches = rugged.branches.each(filter).map do |rugged_ref|
-          begin
-            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
-          rescue Rugged::ReferenceError
-            # Omit invalid branch
-          end
-        end.compact
-
-        sort_branches(branches, sort_by)
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/389
+      def branches(sort_by: nil)
+        branches_filter(sort_by: sort_by)
       end
 
       def reload_rugged
@@ -107,7 +101,10 @@ module Gitlab
         reload_rugged if force_reload
 
         rugged_ref = rugged.branches[name]
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref
+        if rugged_ref
+          target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+          Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+        end
       end
 
       def local_branches(sort_by: nil)
@@ -115,7 +112,7 @@ module Gitlab
           if is_enabled
             gitaly_ref_client.local_branches(sort_by: sort_by)
           else
-            branches(filter: :local, sort_by: sort_by)
+            branches_filter(filter: :local, sort_by: sort_by)
           end
         end
       end
@@ -162,6 +159,8 @@ module Gitlab
       end
 
       # Returns an Array of Tags
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390
       def tags
         rugged.references.each("refs/tags/*").map do |ref|
           message = nil
@@ -174,7 +173,8 @@ module Gitlab
             end
           end
 
-          Gitlab::Git::Tag.new(self, ref.name, ref.target, message)
+          target_commit = Gitlab::Git::Commit.find(self, ref.target)
+          Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message)
         end.sort_by(&:name)
       end
 
@@ -204,13 +204,6 @@ module Gitlab
         branch_names + tag_names
       end
 
-      # Deprecated. Will be removed in 5.2
-      def heads
-        rugged.references.each("refs/heads/*").map do |head|
-          Gitlab::Git::Ref.new(self, head.name, head.target)
-        end.sort_by(&:name)
-      end
-
       def has_commits?
         !empty?
       end
@@ -297,28 +290,6 @@ module Gitlab
         (size.to_f / 1024).round(2)
       end
 
-      # Returns an array of BlobSnippets for files at the specified +ref+ that
-      # contain the +query+ string.
-      def search_files(query, ref = nil)
-        greps = []
-        ref ||= root_ref
-
-        populated_index(ref).each do |entry|
-          # Discard submodules
-          next if submodule?(entry)
-
-          blob = Gitlab::Git::Blob.raw(self, entry[:oid])
-
-          # Skip binary files
-          next if blob.data.encoding == Encoding::ASCII_8BIT
-
-          blob.load_all_data!(self)
-          greps += build_greps(blob.data, query, ref, entry[:path])
-        end
-
-        greps
-      end
-
       # Use the Rugged Walker API to build an array of commits.
       #
       # Usage.
@@ -331,85 +302,10 @@ module Gitlab
       #   )
       #
       def log(options)
-        default_options = {
-          limit: 10,
-          offset: 0,
-          path: nil,
-          follow: false,
-          skip_merges: false,
-          disable_walk: false,
-          after: nil,
-          before: nil
-        }
-
-        options = default_options.merge(options)
-        options[:limit] ||= 0
-        options[:offset] ||= 0
-        actual_ref = options[:ref] || root_ref
-        begin
-          sha = sha_from_ref(actual_ref)
-        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
-          # Return an empty array if the ref wasn't found
-          return []
-        end
-
-        if log_using_shell?(options)
-          log_by_shell(sha, options)
-        else
-          log_by_walk(sha, options)
-        end
-      end
-
-      def log_using_shell?(options)
-        options[:path].present? ||
-          options[:disable_walk] ||
-          options[:skip_merges] ||
-          options[:after] ||
-          options[:before]
-      end
-
-      def log_by_walk(sha, options)
-        walk_options = {
-          show: sha,
-          sort: Rugged::SORT_NONE,
-          limit: options[:limit],
-          offset: options[:offset]
-        }
-        Rugged::Walker.walk(rugged, walk_options).to_a
-      end
-
-      def log_by_shell(sha, options)
-        limit = options[:limit].to_i
-        offset = options[:offset].to_i
-        use_follow_flag = options[:follow] && options[:path].present?
-
-        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
-        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
-        offset_in_ruby = use_follow_flag && options[:offset].present?
-        limit += offset if offset_in_ruby
-
-        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
-        cmd << "--max-count=#{limit}"
-        cmd << '--format=%H'
-        cmd << "--skip=#{offset}" unless offset_in_ruby
-        cmd << '--follow' if use_follow_flag
-        cmd << '--no-merges' if options[:skip_merges]
-        cmd << "--after=#{options[:after].iso8601}" if options[:after]
-        cmd << "--before=#{options[:before].iso8601}" if options[:before]
-        cmd << sha
-
-        # :path can be a string or an array of strings
-        if options[:path].present?
-          cmd << '--'
-          cmd += Array(options[:path])
-        end
-
-        raw_output = IO.popen(cmd) { |io| io.read }
-        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
-
-        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
+        raw_log(options).map { |c| Commit.decorate(c) }
       end
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382
       def count_commits(options)
         cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
         cmd << "--after=#{options[:after].iso8601}" if options[:after]
@@ -454,7 +350,7 @@ module Gitlab
 
       # Counts the amount of commits between `from` and `to`.
       def count_commits_between(from, to)
-        commits_between(from, to).size
+        Commit.between(self, from, to).size
       end
 
       # Returns the SHA of the most recent common ancestor of +from+ and +to+
@@ -553,23 +449,35 @@ module Gitlab
       #   @repository.submodule_url_for('master', 'rack')
       #   # => git@localhost:rack.git
       #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329
       def submodule_url_for(ref, path)
-        if submodules(ref).any?
-          submodule = submodules(ref)[path]
-
-          if submodule
-            submodule['url']
+        Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled|
+          if is_enabled
+            gitaly_submodule_url_for(ref, path)
+          else
+            if submodules(ref).any?
+              submodule = submodules(ref)[path]
+              submodule['url'] if submodule
+            end
           end
         end
       end
 
       # Return total commits count accessible from passed ref
+      #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330
       def commit_count(ref)
-        walker = Rugged::Walker.new(rugged)
-        walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
-        oid = rugged.rev_parse_oid(ref)
-        walker.push(oid)
-        walker.count
+        gitaly_migrate(:commit_count) do |is_enabled|
+          if is_enabled
+            gitaly_commit_client.commit_count(ref)
+          else
+            walker = Rugged::Walker.new(rugged)
+            walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+            oid = rugged.rev_parse_oid(ref)
+            walker.push(oid)
+            walker.count
+          end
+        end
       end
 
       # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or
@@ -770,7 +678,8 @@ module Gitlab
       #   create_branch("other-feature", "master")
       def create_branch(ref, start_point = "HEAD")
         rugged_ref = rugged.branches.create(ref, start_point)
-        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
+        target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
       rescue Rugged::ReferenceError => e
         raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
         raise InvalidRef.new("Invalid reference #{start_point}")
@@ -829,6 +738,7 @@ module Gitlab
       # Ex.
       #   repo.ls_files('master')
       #
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327
       def ls_files(ref)
         actual_ref = ref || root_ref
 
@@ -855,6 +765,7 @@ module Gitlab
         raw_output.compact
       end
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328
       def copy_gitattributes(ref)
         begin
           commit = lookup(ref)
@@ -898,6 +809,103 @@ module Gitlab
 
       private
 
+      # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
+      def branches_filter(filter: nil, sort_by: nil)
+        branches = rugged.branches.each(filter).map do |rugged_ref|
+          begin
+            target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
+            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
+          rescue Rugged::ReferenceError
+            # Omit invalid branch
+          end
+        end.compact
+
+        sort_branches(branches, sort_by)
+      end
+
+      def raw_log(options)
+        default_options = {
+          limit: 10,
+          offset: 0,
+          path: nil,
+          follow: false,
+          skip_merges: false,
+          disable_walk: false,
+          after: nil,
+          before: nil
+        }
+
+        options = default_options.merge(options)
+        options[:limit] ||= 0
+        options[:offset] ||= 0
+        actual_ref = options[:ref] || root_ref
+        begin
+          sha = sha_from_ref(actual_ref)
+        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+          # Return an empty array if the ref wasn't found
+          return []
+        end
+
+        if log_using_shell?(options)
+          log_by_shell(sha, options)
+        else
+          log_by_walk(sha, options)
+        end
+      end
+
+      def log_using_shell?(options)
+        options[:path].present? ||
+          options[:disable_walk] ||
+          options[:skip_merges] ||
+          options[:after] ||
+          options[:before]
+      end
+
+      def log_by_walk(sha, options)
+        walk_options = {
+          show: sha,
+          sort: Rugged::SORT_NONE,
+          limit: options[:limit],
+          offset: options[:offset]
+        }
+        Rugged::Walker.walk(rugged, walk_options).to_a
+      end
+
+      # Gitaly note: JV: although #log_by_shell shells out to Git I think the
+      # complexity is such that we should migrate it as Ruby before trying to
+      # do it in Go.
+      def log_by_shell(sha, options)
+        limit = options[:limit].to_i
+        offset = options[:offset].to_i
+        use_follow_flag = options[:follow] && options[:path].present?
+
+        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
+        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
+        offset_in_ruby = use_follow_flag && options[:offset].present?
+        limit += offset if offset_in_ruby
+
+        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
+        cmd << "--max-count=#{limit}"
+        cmd << '--format=%H'
+        cmd << "--skip=#{offset}" unless offset_in_ruby
+        cmd << '--follow' if use_follow_flag
+        cmd << '--no-merges' if options[:skip_merges]
+        cmd << "--after=#{options[:after].iso8601}" if options[:after]
+        cmd << "--before=#{options[:before].iso8601}" if options[:before]
+        cmd << sha
+
+        # :path can be a string or an array of strings
+        if options[:path].present?
+          cmd << '--'
+          cmd += Array(options[:path])
+        end
+
+        raw_output = IO.popen(cmd) { |io| io.read }
+        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
+
+        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
+      end
+
       # We are trying to deprecate this method because it does a lot of work
       # but it seems to be used only to look up submodule URL's.
       # https://gitlab.com/gitlab-org/gitaly/issues/329
@@ -915,6 +923,18 @@ module Gitlab
         fill_submodule_ids(commit, parser.parse)
       end
 
+      def gitaly_submodule_url_for(ref, path)
+        # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited.
+        commit_object = gitaly_commit_client.tree_entry(ref, path, 1)
+
+        return unless commit_object && commit_object.type == :COMMIT
+
+        gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE)
+        found_module = GitmodulesParser.new(gitmodules.data).parse[path]
+
+        found_module && found_module['url']
+      end
+
       def alternate_object_directories
         Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact
       end
@@ -1057,73 +1077,6 @@ module Gitlab
         index
       end
 
-      # Return an array of BlobSnippets for lines in +file_contents+ that match
-      # +query+
-      def build_greps(file_contents, query, ref, filename)
-        # The file_contents string is potentially huge so we make sure to loop
-        # through it one line at a time. This gives Ruby the chance to GC lines
-        # we are not interested in.
-        #
-        # We need to do a little extra work because we are not looking for just
-        # the lines that matches the query, but also for the context
-        # (surrounding lines). We will use Enumerable#each_cons to efficiently
-        # loop through the lines while keeping surrounding lines on hand.
-        #
-        # First, we turn "foo\nbar\nbaz" into
-        # [
-        #  [nil, -3], [nil, -2], [nil, -1],
-        #  ['foo', 0], ['bar', 1], ['baz', 3],
-        #  [nil, 4], [nil, 5], [nil, 6]
-        # ]
-        lines_with_index = Enumerator.new do |yielder|
-          # Yield fake 'before' lines for the first line of file_contents
-          (-SEARCH_CONTEXT_LINES..-1).each do |i|
-            yielder.yield [nil, i]
-          end
-
-          # Yield the actual file contents
-          count = 0
-          file_contents.each_line do |line|
-            line.chomp!
-            yielder.yield [line, count]
-            count += 1
-          end
-
-          # Yield fake 'after' lines for the last line of file_contents
-          (count + 1..count + SEARCH_CONTEXT_LINES).each do |i|
-            yielder.yield [nil, i]
-          end
-        end
-
-        greps = []
-
-        # Loop through consecutive blocks of lines with indexes
-        lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block|
-          # Get the 'middle' line and index from the block
-          line, _ = line_block[SEARCH_CONTEXT_LINES]
-
-          next unless line && line.match(/#{Regexp.escape(query)}/i)
-
-          # Yay, 'line' contains a match!
-          # Get an array with just the context lines (no indexes)
-          match_with_context = line_block.map(&:first)
-          # Remove 'nil' lines in case we are close to the first or last line
-          match_with_context.compact!
-
-          # Get the line number (1-indexed) of the first context line
-          first_context_line_number = line_block[0][1] + 1
-
-          greps << Gitlab::Git::BlobSnippet.new(
-            ref,
-            match_with_context,
-            first_context_line_number,
-            filename
-          )
-        end
-
-        greps
-      end
-
       # Return the Rugged patches for the diff between +from+ and +to+.
       def diff_patches(from, to, options = {}, *paths)
         options ||= {}
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index a16b0ed76f4..2b5785a1f08 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites.
+
 module Gitlab
   module Git
     class RevList
@@ -15,6 +17,8 @@ module Gitlab
       end
 
       # This methods returns an array of missed references
+      #
+      # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348.
       def missed_ref
         execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"])
       end
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index b5342c3d310..bc4e160dce9 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -1,10 +1,12 @@
+# Gitaly note: JV: no RPC's here.
+#
 module Gitlab
   module Git
     class Tag < Ref
       attr_reader :object_sha
 
-      def initialize(repository, name, target, message = nil)
-        super(repository, name, target)
+      def initialize(repository, name, target, target_commit, message = nil)
+        super(repository, name, target, target_commit)
 
         @message = message
       end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index b9afa05c819..8122ff0e81f 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: needs 1 RPC, migration is in progress.
+
 module Gitlab
   module Git
     class Tree
@@ -10,6 +12,8 @@ module Gitlab
         # Get list of tree objects
         # for repository based on commit sha and path
         # Uses rugged for raw objects
+        #
+        # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320
         def where(repository, sha, path = nil)
           path = nil if path == '' || path == '/'
 
@@ -40,6 +44,8 @@ module Gitlab
           end
         end
 
+        private
+
         # Recursive search of tree id for path
         #
         # Ex.
@@ -80,6 +86,10 @@ module Gitlab
         encode! @name
       end
 
+      def path
+        encode! @path
+      end
+
       def dir?
         type == :tree
       end
diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb
index 7973da2e8f8..4708f22dcb3 100644
--- a/lib/gitlab/git/util.rb
+++ b/lib/gitlab/git/util.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: no RPC's here.
+
 module Gitlab
   module Git
     module Util
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
index 0e87ee30c98..a3c6b21a6a1 100644
--- a/lib/gitlab/git_ref_validator.rb
+++ b/lib/gitlab/git_ref_validator.rb
@@ -1,3 +1,5 @@
+# Gitaly note: JV: does not need to be migrated, works without a repo.
+
 module Gitlab
   module GitRefValidator
     extend self
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index f605c06dfc3..197a94487eb 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -70,12 +70,8 @@ module Gitlab
       params['gitaly_token'].presence || Gitlab.config.gitaly['token']
     end
 
-    def self.enabled?
-      Gitlab.config.gitaly.enabled
-    end
-
     def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
-      return false if !enabled? || status == MigrationStatus::DISABLED
+      return false if status == MigrationStatus::DISABLED
 
       feature = Feature.get("gitaly_#{feature}")
 
diff --git a/lib/gitlab/gitaly_client/blob.rb b/lib/gitlab/gitaly_client/blob.rb
new file mode 100644
index 00000000000..0c398b46a08
--- /dev/null
+++ b/lib/gitlab/gitaly_client/blob.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module GitalyClient
+    class Blob
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+      end
+      
+      def get_blob(oid:, limit:)
+        request = Gitaly::GetBlobRequest.new(
+          repository: @gitaly_repo,
+          oid: oid,
+          limit: limit
+        )
+        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request)
+
+        blob = response.first
+        return unless blob.oid.present?
+
+        data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup }
+
+        Gitlab::Git::Blob.new(
+          id: blob.oid,
+          size: blob.size,
+          data: data,
+          binary: Gitlab::Git::Blob.binary?(data)
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb
index b8877619797..aafc0520664 100644
--- a/lib/gitlab/gitaly_client/commit.rb
+++ b/lib/gitlab/gitaly_client/commit.rb
@@ -56,6 +56,15 @@ module Gitlab
         entry
       end
 
+      def commit_count(ref)
+        request = Gitaly::CountCommitsRequest.new(
+          repository: @gitaly_repo,
+          revision: ref
+        )
+
+        GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count
+      end
+
       private
 
       def commit_diff_request_params(commit, options = {})
diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb
index 6edc69de078..2c17b67d4b7 100644
--- a/lib/gitlab/gitaly_client/ref.rb
+++ b/lib/gitlab/gitaly_client/ref.rb
@@ -72,11 +72,39 @@ module Gitlab
             Gitlab::Git::Branch.new(
               @repository,
               encode!(gitaly_branch.name.dup),
-              gitaly_branch.commit_id
+              gitaly_branch.commit_id,
+              commit_from_local_branches_response(gitaly_branch)
             )
           end
         end
       end
+
+      def commit_from_local_branches_response(response)
+        # Git messages have no encoding enforcements. However, in the UI we only
+        # handle UTF-8, so basically we cross our fingers that the message force
+        # encoded to UTF-8 is readable.
+        message = response.commit_subject.dup.force_encoding('UTF-8')
+
+        # NOTE: For ease of parsing in Gitaly, we have only the subject of
+        # the commit and not the full message. This is ok, since all the
+        # code that uses `local_branches` only cares at most about the
+        # commit message.
+        # TODO: Once gitaly "takes over" Rugged consider separating the
+        # subject from the message to make it clearer when there's one
+        # available but not the other.
+        hash = {
+          id: response.commit_id,
+          message: message,
+          authored_date: Time.at(response.commit_author.date.seconds),
+          author_name: response.commit_author.name,
+          author_email: response.commit_author.email,
+          committed_date: Time.at(response.commit_committer.date.seconds),
+          committer_name: response.commit_committer.name,
+          committer_email: response.commit_committer.email
+        }
+
+        Gitlab::Git::Commit.decorate(hash)
+      end
     end
   end
 end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index e78b7f22e03..bebde857b16 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -35,9 +35,9 @@ module Gitlab
           repository_storages.flat_map do |storage_name|
             tmp_file_path = tmp_file_path(storage_name)
             [
-              operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name),
-              operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name),
-              operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name)
+              operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, -> { storage_stat_test(storage_name) }, shard: storage_name),
+              operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, -> { storage_write_test(tmp_file_path) }, shard: storage_name),
+              operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, -> { storage_read_test(tmp_file_path) }, shard: storage_name)
             ].flatten
           end
         end
@@ -52,7 +52,7 @@ module Gitlab
             ]
           end
         rescue RuntimeError => ex
-          Rails.logger("unexpected error #{ex} when checking #{ok_metric}")
+          Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}")
           [metric(ok_metric, 0, **labels)]
         end
 
diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb
new file mode 100644
index 00000000000..a28658d42d4
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/cache_check.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module HealthChecks
+    module Redis
+      class CacheCheck
+        extend SimpleAbstractCheck
+
+        class << self
+          def check_up
+            check
+          end
+
+          private
+
+          def metric_prefix
+            'redis_cache_ping'
+          end
+
+          def is_successful?(result)
+            result == 'PONG'
+          end
+
+          def check
+            catch_timeout 10.seconds do
+              Gitlab::Redis::Cache.with(&:ping)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb
new file mode 100644
index 00000000000..f97d50d3947
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/queues_check.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module HealthChecks
+    module Redis
+      class QueuesCheck
+        extend SimpleAbstractCheck
+
+        class << self
+          def check_up
+            check
+          end
+
+          private
+
+          def metric_prefix
+            'redis_queues_ping'
+          end
+
+          def is_successful?(result)
+            result == 'PONG'
+          end
+
+          def check
+            catch_timeout 10.seconds do
+              Gitlab::Redis::Queues.with(&:ping)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb
new file mode 100644
index 00000000000..fe4e3c4a3ab
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/redis_check.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module HealthChecks
+    module Redis
+      class RedisCheck
+        extend SimpleAbstractCheck
+
+        class << self
+          private
+
+          def metric_prefix
+            'redis_ping'
+          end
+
+          def is_successful?(result)
+            result == 'PONG'
+          end
+
+          def check
+            ::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
+              ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
+              ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb
new file mode 100644
index 00000000000..e3244392902
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/shared_state_check.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module HealthChecks
+    module Redis
+      class SharedStateCheck
+        extend SimpleAbstractCheck
+
+        class << self
+          def check_up
+            check
+          end
+
+          private
+
+          def metric_prefix
+            'redis_shared_state_ping'
+          end
+
+          def is_successful?(result)
+            result == 'PONG'
+          end
+
+          def check
+            catch_timeout 10.seconds do
+              Gitlab::Redis::SharedState.with(&:ping)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb
deleted file mode 100644
index 57bbe5b3ad0..00000000000
--- a/lib/gitlab/health_checks/redis_check.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-module Gitlab
-  module HealthChecks
-    class RedisCheck
-      extend SimpleAbstractCheck
-
-      class << self
-        private
-
-        def metric_prefix
-          'redis_ping'
-        end
-
-        def is_successful?(result)
-          result == 'PONG'
-        end
-
-        def check
-          catch_timeout 10.seconds do
-            Gitlab::Redis.with(&:ping)
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb
index fbe1645c1b1..3dcb28a193c 100644
--- a/lib/gitlab/health_checks/simple_abstract_check.rb
+++ b/lib/gitlab/health_checks/simple_abstract_check.rb
@@ -20,7 +20,7 @@ module Gitlab
           [
             metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0),
             metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0),
-            metric("#{metric_prefix}_latency", elapsed)
+            metric("#{metric_prefix}_latency_seconds", elapsed)
           ]
         end
       end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 1860352c96d..c8ad3a7a5e0 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -27,6 +27,7 @@ project_tree:
       - :author
       - :events
     - merge_request_diff:
+      - :merge_request_diff_commits
       - :merge_request_diff_files
     - :events
     - :timelogs
diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb
new file mode 100644
index 00000000000..977c05910d3
--- /dev/null
+++ b/lib/gitlab/issuable_metadata.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module IssuableMetadata
+    def issuable_meta_data(issuable_collection, collection_type)
+      # map has to be used here since using pluck or select will
+      # throw an error when ordering issuables by priority which inserts
+      # a new order into the collection.
+      # We cannot use reorder to not mess up the paginated collection.
+      issuable_ids = issuable_collection.map(&:id)
+
+      return {} if issuable_ids.empty?
+
+      issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type)
+      issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type)
+      issuable_merge_requests_count =
+        if collection_type == 'Issue'
+          ::MergeRequestsClosingIssues.count_for_collection(issuable_ids)
+        else
+          []
+        end
+
+      issuable_ids.each_with_object({}) do |id, issuable_meta|
+        downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? }
+        upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? }
+        notes = issuable_note_count.find { |notes| notes.noteable_id == id }
+        merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id }
+
+        issuable_meta[id] = ::Issuable::IssuableMeta.new(
+          upvotes.try(:count).to_i,
+          downvotes.try(:count).to_i,
+          notes.try(:count).to_i,
+          merge_requests.try(:last).to_i
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb
index c56c1a4322f..cdbdfa10d0e 100644
--- a/lib/gitlab/kubernetes.rb
+++ b/lib/gitlab/kubernetes.rb
@@ -76,5 +76,44 @@ module Gitlab
 
       url.to_s
     end
+
+    def to_kubeconfig(url:, namespace:, token:, ca_pem: nil)
+      config = {
+        apiVersion: 'v1',
+        clusters: [
+          name: 'gitlab-deploy',
+          cluster: {
+            server: url
+          }
+        ],
+        contexts: [
+          name: 'gitlab-deploy',
+          context: {
+            cluster: 'gitlab-deploy',
+            namespace: namespace,
+            user: 'gitlab-deploy'
+          }
+        ],
+        'current-context': 'gitlab-deploy',
+        kind: 'Config',
+        users: [
+          {
+            name: 'gitlab-deploy',
+            user: { token: token }
+          }
+        ]
+      }
+
+      kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem
+
+      config.deep_stringify_keys
+    end
+
+    private
+
+    def kubeconfig_embed_ca_pem(config, ca_pem)
+      cluster = config.dig(:clusters, 0, :cluster)
+      cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem)
+    end
   end
 end
diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb
index 5f67e97fa2a..8e57ba831c5 100644
--- a/lib/gitlab/lfs_token.rb
+++ b/lib/gitlab/lfs_token.rb
@@ -18,10 +18,10 @@ module Gitlab
     end
 
     def token
-      Gitlab::Redis.with do |redis|
-        token = redis.get(redis_key)
+      Gitlab::Redis::SharedState.with do |redis|
+        token = redis.get(redis_shared_state_key)
         token ||= Devise.friendly_token(TOKEN_LENGTH)
-        redis.set(redis_key, token, ex: EXPIRY_TIME)
+        redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME)
 
         token
       end
@@ -41,7 +41,7 @@ module Gitlab
 
     private
 
-    def redis_key
+    def redis_shared_state_key
       "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor
     end
   end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index 3503fac40e8..9f432673a6e 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -1,6 +1,6 @@
 require 'yaml'
 require 'json'
-require_relative 'redis' unless defined?(Gitlab::Redis)
+require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues)
 
 module Gitlab
   module MailRoom
@@ -34,11 +34,11 @@ module Gitlab
         config[:idle_timeout] = 60 if config[:idle_timeout].nil?
 
         if config[:enabled] && config[:address]
-          gitlab_redis = Gitlab::Redis.new(rails_env)
-          config[:redis_url] = gitlab_redis.url
+          gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env)
+          config[:redis_url] = gitlab_redis_queues.url
 
-          if gitlab_redis.sentinels?
-            config[:sentinels] = gitlab_redis.sentinels
+          if gitlab_redis_queues.sentinels?
+            config[:sentinels] = gitlab_redis_queues.sentinels
           end
         end
 
diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb
deleted file mode 100644
index b3da360be8f..00000000000
--- a/lib/gitlab/metrics/connection_rack_middleware.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-module Gitlab
-  module Metrics
-    class ConnectionRackMiddleware
-      def initialize(app)
-        @app = app
-      end
-
-      def self.rack_request_count
-        @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count')
-      end
-
-      def self.rack_response_count
-        @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count')
-      end
-
-      def self.rack_uncaught_errors_count
-        @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count')
-      end
-
-      def self.rack_execution_time
-        @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time',
-                                                           {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10])
-      end
-
-      def call(env)
-        method = env['REQUEST_METHOD'].downcase
-        started = Time.now.to_f
-        begin
-          ConnectionRackMiddleware.rack_request_count.increment(method: method)
-
-          status, headers, body = @app.call(env)
-
-          ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status)
-          [status, headers, body]
-        rescue
-          ConnectionRackMiddleware.rack_uncaught_errors_count.increment
-          raise
-        ensure
-          elapsed = Time.now.to_f - started
-          ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
new file mode 100644
index 00000000000..0dc19f31d03
--- /dev/null
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module Metrics
+    class RequestsRackMiddleware
+      def initialize(app)
+        @app = app
+      end
+
+      def self.http_request_total
+        @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count')
+      end
+
+      def self.rack_uncaught_errors_count
+        @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count')
+      end
+
+      def self.http_request_duration_seconds
+        @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time',
+                                                           {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25])
+      end
+
+      def call(env)
+        method = env['REQUEST_METHOD'].downcase
+        started = Time.now.to_f
+        begin
+          RequestsRackMiddleware.http_request_total.increment(method: method)
+
+          status, headers, body = @app.call(env)
+
+          elapsed = Time.now.to_f - started
+          RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed)
+
+          [status, headers, body]
+        rescue
+          RequestsRackMiddleware.rack_uncaught_errors_count.increment
+          raise
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index b3f453e506d..3f2bbd9f6a6 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -101,14 +101,18 @@ module Gitlab
         # Look for a corresponding person with same uid in any of the configured LDAP providers
         Gitlab::LDAP::Config.providers.each do |provider|
           adapter = Gitlab::LDAP::Adapter.new(provider)
-          @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter)
-          # The `uid` might actually be a DN. Try it next.
-          @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter)
+          @ldap_person = find_ldap_person(auth_hash, adapter)
           break if @ldap_person
         end
         @ldap_person
       end
 
+      def find_ldap_person(auth_hash, adapter)
+        by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter)
+        # The `uid` might actually be a DN. Try it next.
+        by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter)
+      end
+
       def ldap_config
         Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person
       end
diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb
index 0d541935bc6..22332474945 100644
--- a/lib/gitlab/otp_key_rotator.rb
+++ b/lib/gitlab/otp_key_rotator.rb
@@ -34,7 +34,7 @@ module Gitlab
 
       write_csv do |csv|
         ActiveRecord::Base.transaction do
-          User.with_two_factor.in_batches do |relation|
+          User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches
             rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt)
             rows.each do |row|
               user = %i[id ciphertext iv salt].zip(row).to_h
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 10eb99fb461..d81f825ef96 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -112,6 +112,7 @@ module Gitlab
     # this group would not be accessible through `/groups/parent/activity` since
     # this would map to the activity-page of its parent.
     GROUP_ROUTES = %w[
+      -
       activity
       analytics
       audit_events
diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb
index 163a40ad306..56112ec2301 100644
--- a/lib/gitlab/performance_bar.rb
+++ b/lib/gitlab/performance_bar.rb
@@ -1,7 +1,34 @@
 module Gitlab
   module PerformanceBar
-    def self.enabled?
-      Feature.enabled?('gitlab_performance_bar')
+    include Gitlab::CurrentSettings
+
+    ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze
+    EXPIRY_TIME = 5.minutes
+
+    def self.enabled?(user = nil)
+      return false unless user && allowed_group_id
+
+      allowed_user_ids.include?(user.id)
+    end
+
+    def self.allowed_group_id
+      current_application_settings.performance_bar_allowed_group_id
+    end
+
+    def self.allowed_user_ids
+      Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do
+        group = Group.find_by_id(allowed_group_id)
+
+        if group
+          GroupMembersFinder.new(group).execute.pluck(:user_id)
+        else
+          []
+        end
+      end
+    end
+
+    def self.expire_allowed_user_ids_cache
+      Rails.cache.delete(ALLOWED_USER_IDS_KEY)
     end
   end
 end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
deleted file mode 100644
index bc5370de32a..00000000000
--- a/lib/gitlab/redis.rb
+++ /dev/null
@@ -1,102 +0,0 @@
-# This file should not have any direct dependency on Rails environment
-# please require all dependencies below:
-require 'active_support/core_ext/hash/keys'
-require 'active_support/core_ext/module/delegation'
-
-module Gitlab
-  class Redis
-    CACHE_NAMESPACE = 'cache:gitlab'.freeze
-    SESSION_NAMESPACE = 'session:gitlab'.freeze
-    SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze
-    MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze
-    DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze
-
-    class << self
-      delegate :params, :url, to: :new
-
-      def with
-        @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
-        @pool.with { |redis| yield redis }
-      end
-
-      def pool_size
-        if Sidekiq.server?
-          # the pool will be used in a multi-threaded context
-          Sidekiq.options[:concurrency] + 5
-        else
-          # probably this is a Unicorn process, so single threaded
-          5
-        end
-      end
-
-      def _raw_config
-        return @_raw_config if defined?(@_raw_config)
-
-        begin
-          @_raw_config = ERB.new(File.read(config_file)).result.freeze
-        rescue Errno::ENOENT
-          @_raw_config = false
-        end
-
-        @_raw_config
-      end
-
-      def config_file
-        ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__)
-      end
-    end
-
-    def initialize(rails_env = nil)
-      @rails_env = rails_env || ::Rails.env
-    end
-
-    def params
-      redis_store_options
-    end
-
-    def url
-      raw_config_hash[:url]
-    end
-
-    def sentinels
-      raw_config_hash[:sentinels]
-    end
-
-    def sentinels?
-      sentinels && !sentinels.empty?
-    end
-
-    private
-
-    def redis_store_options
-      config = raw_config_hash
-      redis_url = config.delete(:url)
-      redis_uri = URI.parse(redis_url)
-
-      if redis_uri.scheme == 'unix'
-        # Redis::Store does not handle Unix sockets well, so let's do it for them
-        config[:path] = redis_uri.path
-        config
-      else
-        redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
-        # order is important here, sentinels must be after the connection keys.
-        # {url: ..., port: ..., sentinels: [...]}
-        redis_hash.merge(config)
-      end
-    end
-
-    def raw_config_hash
-      config_data = fetch_config
-
-      if config_data
-        config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
-      else
-        { url: DEFAULT_REDIS_URL }
-      end
-    end
-
-    def fetch_config
-      self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false
-    end
-  end
-end
diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb
new file mode 100644
index 00000000000..b0da516ff83
--- /dev/null
+++ b/lib/gitlab/redis/cache.rb
@@ -0,0 +1,34 @@
+# please require all dependencies below:
+require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
+
+module Gitlab
+  module Redis
+    class Cache < ::Gitlab::Redis::Wrapper
+      CACHE_NAMESPACE = 'cache:gitlab'.freeze
+      DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze
+      REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze
+      if defined?(::Rails) && ::Rails.root.present?
+        DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze
+      end
+
+      class << self
+        def default_url
+          DEFAULT_REDIS_CACHE_URL
+        end
+
+        def config_file_name
+          # if ENV set for this class, use it even if it points to a file does not exist
+          file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME]
+          return file_name unless file_name.nil?
+
+          # otherwise, if config files exists for this class, use it
+          file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__)
+          return file_name if File.file?(file_name)
+
+          # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb
new file mode 100644
index 00000000000..f9249d05565
--- /dev/null
+++ b/lib/gitlab/redis/queues.rb
@@ -0,0 +1,35 @@
+# please require all dependencies below:
+require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
+
+module Gitlab
+  module Redis
+    class Queues < ::Gitlab::Redis::Wrapper
+      SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze
+      MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze
+      DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze
+      REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze
+      if defined?(::Rails) && ::Rails.root.present?
+        DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze
+      end
+
+      class << self
+        def default_url
+          DEFAULT_REDIS_QUEUES_URL
+        end
+
+        def config_file_name
+          # if ENV set for this class, use it even if it points to a file does not exist
+          file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME]
+          return file_name if file_name
+
+          # otherwise, if config files exists for this class, use it
+          file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__)
+          return file_name if File.file?(file_name)
+
+          # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
new file mode 100644
index 00000000000..395dcf082da
--- /dev/null
+++ b/lib/gitlab/redis/shared_state.rb
@@ -0,0 +1,34 @@
+# please require all dependencies below:
+require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
+
+module Gitlab
+  module Redis
+    class SharedState < ::Gitlab::Redis::Wrapper
+      SESSION_NAMESPACE = 'session:gitlab'.freeze
+      DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze
+      REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze
+      if defined?(::Rails) && ::Rails.root.present?
+        DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze
+      end
+
+      class << self
+        def default_url
+          DEFAULT_REDIS_SHARED_STATE_URL
+        end
+
+        def config_file_name
+          # if ENV set for this class, use it even if it points to a file does not exist
+          file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME]
+          return file_name if file_name
+
+          # otherwise, if config files exists for this class, use it
+          file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__)
+          return file_name if File.file?(file_name)
+
+          # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
new file mode 100644
index 00000000000..c43b37dde74
--- /dev/null
+++ b/lib/gitlab/redis/wrapper.rb
@@ -0,0 +1,135 @@
+# This file should only be used by sub-classes, not directly by any clients of the sub-classes
+# please require all dependencies below:
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/module/delegation'
+
+module Gitlab
+  module Redis
+    class Wrapper
+      DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze
+      REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze
+      if defined?(::Rails) && ::Rails.root.present?
+        DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze
+      end
+
+      class << self
+        delegate :params, :url, to: :new
+
+        def with
+          @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
+          @pool.with { |redis| yield redis }
+        end
+
+        def pool_size
+          # heuristic constant 5 should be a config setting somewhere -- related to CPU count?
+          size = 5
+          if Sidekiq.server?
+            # the pool will be used in a multi-threaded context
+            size += Sidekiq.options[:concurrency]
+          end
+          size
+        end
+
+        def _raw_config
+          return @_raw_config if defined?(@_raw_config)
+
+          @_raw_config =
+            begin
+              if filename = config_file_name
+                ERB.new(File.read(filename)).result.freeze
+              else
+                false
+              end
+            rescue Errno::ENOENT
+              false
+            end
+        end
+
+        def default_url
+          DEFAULT_REDIS_URL
+        end
+
+        def config_file_name
+          # if ENV set for wrapper class, use it even if it points to a file does not exist
+          file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME]
+          return file_name unless file_name.nil?
+
+          # otherwise, if config files exists for wrapper class, use it
+          file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__)
+          return file_name if File.file?(file_name)
+
+          # nil will force use of DEFAULT_REDIS_URL when config file is absent
+          nil
+        end
+      end
+
+      def initialize(rails_env = nil)
+        @rails_env = rails_env || ::Rails.env
+      end
+
+      def params
+        redis_store_options
+      end
+
+      def url
+        raw_config_hash[:url]
+      end
+
+      def sentinels
+        raw_config_hash[:sentinels]
+      end
+
+      def sentinels?
+        sentinels && !sentinels.empty?
+      end
+
+      private
+
+      def redis_store_options
+        config = raw_config_hash
+        redis_url = config.delete(:url)
+        redis_uri = URI.parse(redis_url)
+
+        if redis_uri.scheme == 'unix'
+          # Redis::Store does not handle Unix sockets well, so let's do it for them
+          config[:path] = redis_uri.path
+          query = redis_uri.query
+          unless query.nil?
+            queries = CGI.parse(redis_uri.query)
+            db_numbers = queries["db"] if queries.key?("db")
+            config[:db] = db_numbers[0].to_i if db_numbers.any?
+          end
+          config
+        else
+          redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
+          # order is important here, sentinels must be after the connection keys.
+          # {url: ..., port: ..., sentinels: [...]}
+          redis_hash.merge(config)
+        end
+      end
+
+      def raw_config_hash
+        config_data = fetch_config
+
+        if config_data
+          config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
+        else
+          { url: self.class.default_url }
+        end
+      end
+
+      def fetch_config
+        return false unless self.class._raw_config
+
+        yaml = YAML.load(self.class._raw_config)
+
+        # If the file has content but it's invalid YAML, `load` returns false
+        if yaml
+          yaml.fetch(@rails_env, false)
+        else
+          false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index b706434217d..c1ee20b6977 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -19,14 +19,6 @@ module Gitlab
       "It must start with letter, digit, emoji or '_'."
     end
 
-    def file_name_regex
-      @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze
-    end
-
-    def file_name_regex_message
-      "can contain only letters, digits, '_', '-', '@', '+' and '.'."
-    end
-
     def container_registry_reference_regex
       Gitlab::PathRegex.git_reference_regex
     end
@@ -38,8 +30,12 @@ module Gitlab
       @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z}
     end
 
+    def environment_name_regex_chars
+      'a-zA-Z0-9_/\\$\\{\\}\\. -'
+    end
+
     def environment_name_regex
-      @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
+      @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze
     end
 
     def environment_name_regex_message
diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb
deleted file mode 100644
index 36d1a8a6f64..00000000000
--- a/lib/gitlab/routes/legacy_builds.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-module Gitlab
-  module Routes
-    class LegacyBuilds
-      def initialize(map)
-        @map = map
-      end
-
-      def draw
-        @map.instance_eval do
-          resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
-            collection do
-              resources :artifacts, only: [], controller: 'build_artifacts' do
-                collection do
-                  get :latest_succeeded,
-                    path: '*ref_name_and_path',
-                    format: false
-                end
-              end
-            end
-
-            member do
-              get :raw
-            end
-
-            resource :artifacts, only: [], controller: 'build_artifacts' do
-              get :download
-              get :browse, path: 'browse(/*path)', format: false
-              get :file, path: 'file/*path', format: false
-              get :raw, path: 'raw/*path', format: false
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb
index 632e2d87500..e57890f1143 100644
--- a/lib/gitlab/routing.rb
+++ b/lib/gitlab/routing.rb
@@ -2,10 +2,35 @@ module Gitlab
   module Routing
     extend ActiveSupport::Concern
 
+    mattr_accessor :_includers
+    self._includers = []
+
     included do
+      Gitlab::Routing.includes_helpers(self)
+
       include Gitlab::Routing.url_helpers
     end
 
+    def self.includes_helpers(klass)
+      self._includers << klass
+    end
+
+    def self.add_helpers(mod)
+      url_helpers.include mod
+      url_helpers.extend mod
+
+      GitlabRoutingHelper.include mod
+      GitlabRoutingHelper.extend mod
+
+      app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module
+      app_url_helpers.include mod
+      app_url_helpers.extend mod
+
+      _includers.each do |klass|
+        klass.include mod
+      end
+    end
+
     # Returns the URL helpers Module.
     #
     # This method caches the output as Rails' "url_helpers" method creates an
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 0baea092e6a..4366ff336ef 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -1,3 +1,6 @@
+# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository.
+# SSH key operations are not part of Gitaly so will never be migrated.
+
 require 'securerandom'
 
 module Gitlab
@@ -68,6 +71,7 @@ module Gitlab
     # Ex.
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def add_repository(storage, name)
       gitlab_shell_fast_execute([gitlab_shell_projects_path,
                                  'add-project', storage, "#{name}.git"])
@@ -81,6 +85,7 @@ module Gitlab
     # Ex.
     #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def import_repository(storage, name, url)
       # Timeout should be less than 900 ideally, to prevent the memory killer
       # to silently kill the process without knowing we are timing out here.
@@ -99,6 +104,7 @@ module Gitlab
     # Ex.
     #   fetch_remote("gitlab/gitlab-ci", "upstream")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def fetch_remote(storage, name, remote, forced: false, no_tags: false)
       args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
       args << '--force' if forced
@@ -115,6 +121,7 @@ module Gitlab
     # Ex.
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def mv_repository(storage, path, new_path)
       gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
                                  storage, "#{path}.git", "#{new_path}.git"])
@@ -129,6 +136,7 @@ module Gitlab
     # Ex.
     #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
     #
+    # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
     def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
       gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project',
                                  forked_from_storage, "#{path}.git", forked_to_storage,
@@ -143,6 +151,7 @@ module Gitlab
     # Ex.
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def remove_repository(storage, name)
       gitlab_shell_fast_execute([gitlab_shell_projects_path,
                                  'rm-project', storage, "#{name}.git"])
@@ -194,6 +203,7 @@ module Gitlab
     # Ex.
     #   add_namespace("/path/to/storage", "gitlab")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def add_namespace(storage, name)
       path = full_path(storage, name)
       FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
@@ -207,6 +217,7 @@ module Gitlab
     # Ex.
     #   rm_namespace("/path/to/storage", "gitlab")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def rm_namespace(storage, name)
       FileUtils.rm_r(full_path(storage, name), force: true)
     end
@@ -216,6 +227,7 @@ module Gitlab
     # Ex.
     #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def mv_namespace(storage, old_name, new_name)
       return false if exists?(storage, new_name) || !exists?(storage, old_name)
 
@@ -241,6 +253,7 @@ module Gitlab
     #   exists?(storage, 'gitlab')
     #   exists?(storage, 'gitlab/cookies.git')
     #
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def exists?(storage, dir_name)
       File.exist?(full_path(storage, dir_name))
     end
diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb
index 27696436574..e13808a2720 100644
--- a/lib/gitlab/slash_commands/presenters/base.rb
+++ b/lib/gitlab/slash_commands/presenters/base.rb
@@ -2,7 +2,7 @@ module Gitlab
   module SlashCommands
     module Presenters
       class Base
-        include Gitlab::Routing.url_helpers
+        include Gitlab::Routing
 
         def initialize(resource = nil)
           @resource = resource
diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb
new file mode 100644
index 00000000000..5e89e12b2b1
--- /dev/null
+++ b/lib/gitlab/sql/glob.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module SQL
+    module Glob
+      extend self
+
+      # Convert a simple glob pattern with wildcard (*) to SQL LIKE pattern
+      # with SQL expression
+      def to_like(pattern)
+        <<~SQL
+          REPLACE(REPLACE(REPLACE(#{pattern},
+                                  #{q('%')}, #{q('\\%')}),
+                          #{q('_')}, #{q('\\_')}),
+                  #{q('*')}, #{q('%')})
+        SQL
+      end
+
+      def q(string)
+        ActiveRecord::Base.connection.quote(string)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 23af9318d1a..824e2d7251f 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -1,6 +1,6 @@
 module Gitlab
   class UrlBuilder
-    include Gitlab::Routing.url_helpers
+    include Gitlab::Routing
     include GitlabRoutingHelper
     include ActionView::RecordIdentifier
 
@@ -23,9 +23,9 @@ module Gitlab
       when WikiPage
         wiki_page_url
       when ProjectSnippet
-        project_snippet_url(object)
+        project_snippet_url(object.project, object)
       when Snippet
-        personal_snippet_url(object)
+        snippet_url(object)
       else
         raise NotImplementedError.new("No URL builder defined for #{object.class}")
       end
@@ -52,26 +52,24 @@ module Gitlab
         commit_url(id: object.commit_id, anchor: dom_id(object))
 
       elsif object.for_issue?
-        issue = Issue.find(object.noteable_id)
-        issue_url(issue, anchor: dom_id(object))
+        issue_url(object.noteable, anchor: dom_id(object))
 
       elsif object.for_merge_request?
-        merge_request = MergeRequest.find(object.noteable_id)
-        merge_request_url(merge_request, anchor: dom_id(object))
+        merge_request_url(object.noteable, anchor: dom_id(object))
 
       elsif object.for_snippet?
-        snippet = Snippet.find(object.noteable_id)
+        snippet = object.noteable
 
         if snippet.is_a?(PersonalSnippet)
           snippet_url(snippet, anchor: dom_id(object))
         else
-          project_snippet_url(snippet, anchor: dom_id(object))
+          project_snippet_url(snippet.project, snippet, anchor: dom_id(object))
         end
       end
     end
 
     def wiki_page_url
-      namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug)
+      project_wiki_url(object.wiki.project, object.slug)
     end
   end
 end
diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb
index eb36ab9fded..125488536e1 100644
--- a/lib/gitlab/user_activities.rb
+++ b/lib/gitlab/user_activities.rb
@@ -6,13 +6,13 @@ module Gitlab
     BATCH_SIZE = 500
 
     def self.record(key, time = Time.now)
-      Gitlab::Redis.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
         redis.hset(KEY, key, time.to_i)
       end
     end
 
     def delete(*keys)
-      Gitlab::Redis.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
         redis.hdel(KEY, keys)
       end
     end
@@ -21,7 +21,7 @@ module Gitlab
       cursor = 0
       loop do
         cursor, pairs =
-          Gitlab::Redis.with do |redis|
+          Gitlab::Redis::SharedState.with do |redis|
             redis.hscan(KEY, cursor, count: BATCH_SIZE)
           end
 
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index f96ee69096d..916ef365d78 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -25,27 +25,25 @@ module Gitlab
           RepoPath: repo_path
         }
 
-        if Gitlab.config.gitaly.enabled
-          server = {
-            address: Gitlab::GitalyClient.address(project.repository_storage),
-            token: Gitlab::GitalyClient.token(project.repository_storage)
-          }
-          params[:Repository] = repository.gitaly_repository.to_h
-
-          feature_enabled = case action.to_s
-                            when 'git_receive_pack'
-                              Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
-                            when 'git_upload_pack'
-                              Gitlab::GitalyClient.feature_enabled?(:post_upload_pack)
-                            when 'info_refs'
-                              true
-                            else
-                              raise "Unsupported action: #{action}"
-                            end
-          if feature_enabled
-            params[:GitalyAddress] = server[:address] # This field will be deprecated
-            params[:GitalyServer] = server
-          end
+        server = {
+          address: Gitlab::GitalyClient.address(project.repository_storage),
+          token: Gitlab::GitalyClient.token(project.repository_storage)
+        }
+        params[:Repository] = repository.gitaly_repository.to_h
+
+        feature_enabled = case action.to_s
+                          when 'git_receive_pack'
+                            Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
+                          when 'git_upload_pack'
+                            Gitlab::GitalyClient.feature_enabled?(:post_upload_pack)
+                          when 'info_refs'
+                            true
+                          else
+                            raise "Unsupported action: #{action}"
+                          end
+        if feature_enabled
+          params[:GitalyAddress] = server[:address] # This field will be deprecated
+          params[:GitalyServer] = server
         end
 
         params
@@ -64,10 +62,21 @@ module Gitlab
       end
 
       def send_git_blob(repository, blob)
-        params = {
-          'RepoPath' => repository.path_to_repo,
-          'BlobId' => blob.id
-        }
+        params = if Gitlab::GitalyClient.feature_enabled?(:project_raw_show)
+                   {
+                     'GitalyServer' => gitaly_server_hash(repository),
+                     'GetBlobRequest' => {
+                       repository: repository.gitaly_repository.to_h,
+                       oid: blob.id,
+                       limit: -1
+                     }
+                   }
+                 else
+                   {
+                     'RepoPath' => repository.path_to_repo,
+                     'BlobId' => blob.id
+                   }
+                 end
 
         [
           SEND_DATA_HEADER,
@@ -178,7 +187,7 @@ module Gitlab
       end
 
       def set_key_and_notify(key, value, expire: nil, overwrite: true)
-        Gitlab::Redis.with do |redis|
+        Gitlab::Redis::Queues.with do |redis|
           result = redis.set(key, value, ex: expire, nx: !overwrite)
           if result
             redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}")
@@ -194,6 +203,13 @@ module Gitlab
       def encode(hash)
         Base64.urlsafe_encode64(JSON.dump(hash))
       end
+
+      def gitaly_server_hash(repository)
+        {
+          address: Gitlab::GitalyClient.address(repository.project.repository_storage),
+          token: Gitlab::GitalyClient.token(repository.project.repository_storage)
+        }
+      end
     end
   end
 end
diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb
index 99f9c2c9b04..7cfe76b7b71 100644
--- a/lib/peek/rblineprof/custom_controller_helpers.rb
+++ b/lib/peek/rblineprof/custom_controller_helpers.rb
@@ -41,9 +41,14 @@ module Peek
             ]
           end.sort_by{ |a,b,c,d,e,f| -f }
 
-          output = ''
-          per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort|
+          output = "<div class='modal-dialog modal-full'><div class='modal-content'>"
+          output << "<div class='modal-header'>"
+          output << "<button class='close btn btn-link btn-sm' type='button' data-dismiss='modal'>X</button>"
+          output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>"
+          output << "</div>"
+          output << "<div class='modal-body'>"
 
+          per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort|
             output << "<div class='peek-rblineprof-file'><div class='heading'>"
 
             show_src = file_sort > min
@@ -86,11 +91,32 @@ module Peek
             output << "</div></div>" # .data then .peek-rblineprof-file
           end
 
-          response.body += "<div class='peek-rblineprof-modal' id='line-profile'>#{output}</div>".html_safe
+          output << "</div></div></div>"
+
+          response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output}</div>".html_safe
         end
 
         ret
       end
+
+      private
+
+      def human_description(lineprofiler_param)
+        case lineprofiler_param
+        when 'app'
+          'app/ & lib/'
+        when 'views'
+          'app/view/'
+        when 'gems'
+          'vendor/gems'
+        when 'all'
+          'everything in Rails.root'
+        when 'stdlib'
+          'everything in the Ruby standard library'
+        else
+          'app/, config/, lib/, vendor/ & plugin/'
+        end
+      end
     end
   end
 end
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 125a3d560d6..564aa141952 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -5,12 +5,12 @@ namespace :cache do
 
     desc "GitLab | Clear redis cache"
     task redis: :environment do
-      Gitlab::Redis.with do |redis|
+      Gitlab::Redis::Cache.with do |redis|
         cursor = REDIS_SCAN_START_STOP
         loop do
           cursor, keys = redis.scan(
             cursor,
-            match: "#{Gitlab::Redis::CACHE_NAMESPACE}*",
+            match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*",
             count: REDIS_CLEAR_BATCH_SIZE
           )
 
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index e3883278886..e9fb6a008b0 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -42,8 +42,7 @@ namespace :gitlab do
       http_clone_url = project.http_url_to_repo
       ssh_clone_url  = project.ssh_url_to_repo
 
-      omniauth_providers = Gitlab.config.omniauth.providers
-      omniauth_providers.map! { |provider| provider['name'] }
+      omniauth_providers = Gitlab.config.omniauth.providers.map { |provider| provider['name'] }
 
       puts ""
       puts "GitLab information".color(:yellow)