Merge remote-tracking branch 'origin/master' into 38759-fetch-available-parameters-directly-from-gke-when-creating-a-cluster

79 files changed, 1099 insertions, 569 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 5139e869c71..de20b2b8e67 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -8,14 +8,15 @@ module API
     PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze
     COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze
 
-    use GrapeLogging::Middleware::RequestLogger,
-        logger: Logger.new(LOG_FILENAME),
-        formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new,
-        include: [
-          GrapeLogging::Loggers::FilterParameters.new,
-          GrapeLogging::Loggers::ClientEnv.new,
-          Gitlab::GrapeLogging::Loggers::UserLogger.new
-        ]
+    insert_before Grape::Middleware::Error,
+                  GrapeLogging::Middleware::RequestLogger,
+                  logger: Logger.new(LOG_FILENAME),
+                  formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new,
+                  include: [
+                    GrapeLogging::Loggers::FilterParameters.new,
+                    GrapeLogging::Loggers::ClientEnv.new,
+                    Gitlab::GrapeLogging::Loggers::UserLogger.new
+                  ]
 
     allow_access_with_scope :api
     prefix :api
@@ -139,6 +140,7 @@ module API
     mount ::API::Keys
     mount ::API::Labels
     mount ::API::Lint
+    mount ::API::Markdown
     mount ::API::Members
     mount ::API::MergeRequestDiffs
     mount ::API::MergeRequests
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 25d78fc761d..174c5af91d5 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -125,7 +125,7 @@ module API
         # (fixed in https://github.com/rails/rails/pull/25976).
         project.tags.map(&:name).sort
       end
-      expose :ssh_url_to_repo, :http_url_to_repo, :web_url
+      expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url
       expose :avatar_url do |project, options|
         project.avatar_url(only_path: false)
       end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 92e3d5cc10a..03b6b30a0d8 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -165,9 +165,12 @@ module API
         group = find_group!(params[:id])
         authorize! :admin_group, group
 
+        Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/46285')
         destroy_conditionally!(group) do |group|
-          ::Groups::DestroyService.new(group, current_user).execute
+          ::Groups::DestroyService.new(group, current_user).async_execute
         end
+
+        accepted!
       end
 
       desc 'Get a list of projects in this group.' do
diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb
index 09805049169..3308212216e 100644
--- a/lib/api/helpers/pagination.rb
+++ b/lib/api/helpers/pagination.rb
@@ -2,67 +2,240 @@ module API
   module Helpers
     module Pagination
       def paginate(relation)
-        relation = add_default_order(relation)
+        strategy = if params[:pagination] == 'keyset' && Feature.enabled?('api_keyset_pagination')
+                     KeysetPaginationStrategy
+                   else
+                     DefaultPaginationStrategy
+                   end
 
-        relation.page(params[:page]).per(params[:per_page]).tap do |data|
-          add_pagination_headers(data)
-        end
+        strategy.new(self).paginate(relation)
       end
 
-      private
+      class KeysetPaginationInfo
+        attr_reader :relation, :request_context
 
-      def add_pagination_headers(paginated_data)
-        header 'X-Per-Page',    paginated_data.limit_value.to_s
-        header 'X-Page',        paginated_data.current_page.to_s
-        header 'X-Next-Page',   paginated_data.next_page.to_s
-        header 'X-Prev-Page',   paginated_data.prev_page.to_s
-        header 'Link',          pagination_links(paginated_data)
+        def initialize(relation, request_context)
+          # This is because it's rather complex to support multiple values with possibly different sort directions
+          # (and we don't need this in the API)
+          if relation.order_values.size > 1
+            raise "Pagination only supports ordering by a single column." \
+              "The following columns were given: #{relation.order_values.map { |v| v.expr.name }}"
+          end
 
-        return if data_without_counts?(paginated_data)
+          @relation = relation
+          @request_context = request_context
+        end
 
-        header 'X-Total',       paginated_data.total_count.to_s
-        header 'X-Total-Pages', total_pages(paginated_data).to_s
-      end
+        def fields
+          keys.zip(values).reject { |_, v| v.nil? }.to_h
+        end
 
-      def pagination_links(paginated_data)
-        request_url = request.url.split('?').first
-        request_params = params.clone
-        request_params[:per_page] = paginated_data.limit_value
+        def column_for_order_by(relation)
+          relation.order_values.first&.expr&.name
+        end
 
-        links = []
+        # Sort direction (`:asc` or `:desc`)
+        def sort
+          @sort ||= if order_by_primary_key?
+                      # Default order is by id DESC
+                      :desc
+                    else
+                      # API defaults to DESC order if param `sort` not present
+                      request_context.params[:sort]&.to_sym || :desc
+                    end
+        end
 
-        request_params[:page] = paginated_data.prev_page
-        links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page]
+        # Do we only sort by primary key?
+        def order_by_primary_key?
+          keys.size == 1 && keys.first == primary_key
+        end
 
-        request_params[:page] = paginated_data.next_page
-        links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page]
+        def primary_key
+          relation.model.primary_key.to_sym
+        end
 
-        request_params[:page] = 1
-        links << %(<#{request_url}?#{request_params.to_query}>; rel="first")
+        def sort_ascending?
+          sort == :asc
+        end
 
-        unless data_without_counts?(paginated_data)
-          request_params[:page] = total_pages(paginated_data)
-          links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
+        # Build hash of request parameters for a given record (relevant to pagination)
+        def params_for(record)
+          return {} unless record
+
+          keys.each_with_object({}) do |key, h|
+            h["ks_prev_#{key}".to_sym] = record.attributes[key.to_s]
+          end
         end
 
-        links.join(', ')
-      end
+        private
+
+        # All values present in request parameters that correspond to #keys.
+        def values
+          @values ||= keys.map do |key|
+            request_context.params["ks_prev_#{key}".to_sym]
+          end
+        end
 
-      def total_pages(paginated_data)
-        # Ensure there is in total at least 1 page
-        [paginated_data.total_pages, 1].max
+        # All keys relevant to pagination.
+        # This always includes the primary key. Optionally, the `order_by` key is prepended.
+        def keys
+          @keys ||= [column_for_order_by(relation), primary_key].compact.uniq
+        end
       end
 
-      def add_default_order(relation)
-        if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty?
-          relation = relation.order(:id)
+      class KeysetPaginationStrategy
+        attr_reader :request_context
+        delegate :params, :header, :request, to: :request_context
+
+        def initialize(request_context)
+          @request_context = request_context
+        end
+
+        def paginate(relation)
+          pagination = KeysetPaginationInfo.new(relation, request_context)
+
+          paged_relation = relation.limit(per_page)
+
+          if conds = conditions(pagination)
+            paged_relation = paged_relation.where(*conds)
+          end
+
+          # In all cases: sort by primary key (possibly in addition to another sort column)
+          paged_relation = paged_relation.order(pagination.primary_key => pagination.sort)
+
+          add_default_pagination_headers
+
+          if last_record = paged_relation.last
+            next_page_params = pagination.params_for(last_record)
+            add_navigation_links(next_page_params)
+          end
+
+          paged_relation
+        end
+
+        private
+
+        def conditions(pagination)
+          fields = pagination.fields
+
+          return nil if fields.empty?
+
+          placeholder = fields.map { '?' }
+
+          comp = if pagination.sort_ascending?
+                   '>'
+                 else
+                   '<'
+                 end
+
+          [
+            # Row value comparison:
+            # (A, B) < (a, b) <=> (A < a) OR (A = a AND B < b)
+            #     <=> A <= a AND ((A < a) OR (A = a AND B < b))
+            "(#{fields.keys.join(',')}) #{comp} (#{placeholder.join(',')})",
+            *fields.values
+          ]
+        end
+
+        def per_page
+          params[:per_page]
+        end
+
+        def add_default_pagination_headers
+          header 'X-Per-Page',    per_page.to_s
+        end
+
+        def add_navigation_links(next_page_params)
+          header 'X-Next-Page', page_href(next_page_params)
+          header 'Link', link_for('next', next_page_params)
         end
 
-        relation
+        def page_href(next_page_params)
+          request_url = request.url.split('?').first
+          request_params = params.dup
+          request_params[:per_page] = per_page
+
+          request_params.merge!(next_page_params) if next_page_params
+
+          "#{request_url}?#{request_params.to_query}"
+        end
+
+        def link_for(rel, next_page_params)
+          %(<#{page_href(next_page_params)}>; rel="#{rel}")
+        end
       end
 
-      def data_without_counts?(paginated_data)
-        paginated_data.is_a?(Kaminari::PaginatableWithoutCount)
+      class DefaultPaginationStrategy
+        attr_reader :request_context
+        delegate :params, :header, :request, to: :request_context
+
+        def initialize(request_context)
+          @request_context = request_context
+        end
+
+        def paginate(relation)
+          relation = add_default_order(relation)
+
+          relation.page(params[:page]).per(params[:per_page]).tap do |data|
+            add_pagination_headers(data)
+          end
+        end
+
+        private
+
+        def add_default_order(relation)
+          if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty?
+            relation = relation.order(:id)
+          end
+
+          relation
+        end
+
+        def add_pagination_headers(paginated_data)
+          header 'X-Per-Page',    paginated_data.limit_value.to_s
+          header 'X-Page',        paginated_data.current_page.to_s
+          header 'X-Next-Page',   paginated_data.next_page.to_s
+          header 'X-Prev-Page',   paginated_data.prev_page.to_s
+          header 'Link',          pagination_links(paginated_data)
+
+          return if data_without_counts?(paginated_data)
+
+          header 'X-Total',       paginated_data.total_count.to_s
+          header 'X-Total-Pages', total_pages(paginated_data).to_s
+        end
+
+        def pagination_links(paginated_data)
+          request_url = request.url.split('?').first
+          request_params = params.clone
+          request_params[:per_page] = paginated_data.limit_value
+
+          links = []
+
+          request_params[:page] = paginated_data.prev_page
+          links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page]
+
+          request_params[:page] = paginated_data.next_page
+          links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page]
+
+          request_params[:page] = 1
+          links << %(<#{request_url}?#{request_params.to_query}>; rel="first")
+
+          unless data_without_counts?(paginated_data)
+            request_params[:page] = total_pages(paginated_data)
+            links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
+          end
+
+          links.join(', ')
+        end
+
+        def total_pages(paginated_data)
+          # Ensure there is in total at least 1 page
+          [paginated_data.total_pages, 1].max
+        end
+
+        def data_without_counts?(paginated_data)
+          paginated_data.is_a?(Kaminari::PaginatableWithoutCount)
+        end
       end
     end
   end
diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb
index 7f4d6e58b34..a2cbed30229 100644
--- a/lib/api/helpers/related_resources_helpers.rb
+++ b/lib/api/helpers/related_resources_helpers.rb
@@ -13,9 +13,14 @@ module API
 
       def expose_url(path)
         url_options = Gitlab::Application.routes.default_url_options
-        protocol, host, port = url_options.slice(:protocol, :host, :port).values
+        protocol, host, port, script_name = url_options.values_at(:protocol, :host, :port, :script_name)
 
-        URI::Generic.build(scheme: protocol, host: host, port: port, path: path).to_s
+        # Using a blank component at the beginning of the join we ensure
+        # that the resulted path will start with '/'. If the resulted path
+        # does not start with '/', URI::Generic#build will fail
+        path_with_script_name = File.join('', [script_name, path].select(&:present?))
+
+        URI::Generic.build(scheme: protocol, host: host, port: port, path: path_with_script_name).to_s
       end
 
       private
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 6b72caea8fd..a3dac36b8b6 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -113,7 +113,7 @@ module API
         {
           api_version: API.version,
           gitlab_version: Gitlab::VERSION,
-          gitlab_rev: Gitlab::REVISION,
+          gitlab_rev: Gitlab.revision,
           redis: redis_ping
         }
       end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 12ff2a1398b..6d75e8817c4 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -2,7 +2,7 @@ module API
   class Issues < Grape::API
     include PaginationParams
 
-    before { authenticate! }
+    before { authenticate_non_get! }
 
     helpers ::Gitlab::IssuableMetadata
 
@@ -13,6 +13,7 @@ module API
         args.delete(:id)
         args[:milestone_title] = args.delete(:milestone)
         args[:label_name] = args.delete(:labels)
+        args[:scope] = args[:scope].underscore if args[:scope]
 
         issues = IssuesFinder.new(current_user, args).execute
           .preload(:assignees, :labels, :notes, :timelogs)
@@ -36,8 +37,8 @@ module API
         optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time'
         optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID'
         optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID'
-        optional :scope, type: String, values: %w[created-by-me assigned-to-me all],
-                         desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`'
+        optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all],
+                         desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`'
         optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji'
         use :pagination
       end
@@ -66,10 +67,11 @@ module API
         optional :state, type: String, values: %w[opened closed all], default: 'all',
                          desc: 'Return opened, closed, or all issues'
         use :issues_params
-        optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me',
-                         desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`'
+        optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], default: 'created_by_me',
+                         desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`'
       end
       get do
+        authenticate! unless params[:scope] == 'all'
         issues = paginate(find_issues)
 
         options = {
diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb
new file mode 100644
index 00000000000..b9ed68aa584
--- /dev/null
+++ b/lib/api/markdown.rb
@@ -0,0 +1,33 @@
+module API
+  class Markdown < Grape::API
+    params do
+      requires :text, type: String, desc: "The markdown text to render"
+      optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown"
+      optional :project, type: String, desc: "The full path of a project to use as the context when creating references using GitLab Flavored Markdown"
+    end
+    resource :markdown do
+      desc "Render markdown text" do
+        detail "This feature was introduced in GitLab 11.0."
+      end
+      post do
+        # Explicitly set CommonMark as markdown engine to use.
+        # Remove this set when https://gitlab.com/gitlab-org/gitlab-ce/issues/43011 is done.
+        context = { markdown_engine: :common_mark, only_path: false }
+
+        if params[:project]
+          project = Project.find_by_full_path(params[:project])
+
+          not_found!("Project") unless can?(current_user, :read_project, project)
+
+          context[:project] = project
+        else
+          context[:skip_project_check] = true
+        end
+
+        context[:pipeline] = params[:gfm] ? :full : :plain_markdown
+
+        { html: Banzai.render(params[:text], context) }
+      end
+    end
+  end
+end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index d4cc18f622b..bc4df16e3a8 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -38,6 +38,7 @@ module API
 
         args[:milestone_title] = args.delete(:milestone)
         args[:label_name] = args.delete(:labels)
+        args[:scope] = args[:scope].underscore if args[:scope]
 
         merge_requests = MergeRequestsFinder.new(current_user, args).execute
                            .reorder(args[:order_by] => args[:sort])
@@ -79,8 +80,8 @@ module API
         optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request'
         optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID'
         optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID'
-        optional :scope, type: String, values: %w[created-by-me assigned-to-me all],
-                         desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`'
+        optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all],
+                         desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`'
         optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji'
         optional :source_branch, type: String, desc: 'Return merge requests with the given source branch'
         optional :target_branch, type: String, desc: 'Return merge requests with the given target branch'
@@ -95,8 +96,8 @@ module API
       end
       params do
         use :merge_requests_params
-        optional :scope, type: String, values: %w[created-by-me assigned-to-me all], default: 'created-by-me',
-                         desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`'
+        optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], default: 'created_by_me',
+                         desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`'
       end
       get do
         authenticate! unless params[:scope] == 'all'
diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb
index c570eace862..a8eb137e46a 100644
--- a/lib/api/milestone_responses.rb
+++ b/lib/api/milestone_responses.rb
@@ -24,7 +24,7 @@ module API
           optional :state_event, type: String, values: %w[close activate],
                                  desc: 'The state event of the milestone '
           use :optional_params
-          at_least_one_of :title, :description, :due_date, :state_event
+          at_least_one_of :title, :description, :start_date, :due_date, :state_event
         end
 
         def list_milestones_for(parent)
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 649feba1036..a7f1cb1131f 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -149,6 +149,7 @@ module API
       end
       patch '/:id/trace' do
         job = authenticate_job!
+        forbidden!('Job is not running') unless job.running?
 
         error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
         content_range = request.headers['Content-Range']
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
index 5f2a9567605..5cb96d467c0 100644
--- a/lib/api/runners.rb
+++ b/lib/api/runners.rb
@@ -14,7 +14,7 @@ module API
         use :pagination
       end
       get do
-        runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: %w(specific shared))
+        runners = filter_runners(current_user.ci_owned_runners, params[:scope], without: %w(specific shared))
         present paginate(runners), with: Entities::Runner
       end
 
@@ -184,40 +184,35 @@ module API
       def authenticate_show_runner!(runner)
         return if runner.is_shared || current_user.admin?
 
-        forbidden!("No access granted") unless user_can_access_runner?(runner)
+        forbidden!("No access granted") unless can?(current_user, :read_runner, runner)
       end
 
       def authenticate_update_runner!(runner)
         return if current_user.admin?
 
-        forbidden!("Runner is shared") if runner.is_shared?
-        forbidden!("No access granted") unless user_can_access_runner?(runner)
+        forbidden!("No access granted") unless can?(current_user, :update_runner, runner)
       end
 
       def authenticate_delete_runner!(runner)
         return if current_user.admin?
 
-        forbidden!("Runner is shared") if runner.is_shared?
         forbidden!("Runner associated with more than one project") if runner.projects.count > 1
-        forbidden!("No access granted") unless user_can_access_runner?(runner)
+        forbidden!("No access granted") unless can?(current_user, :delete_runner, runner)
       end
 
       def authenticate_enable_runner!(runner)
-        forbidden!("Runner is shared") if runner.is_shared?
-        forbidden!("Runner is locked") if runner.locked?
+        forbidden!("Runner is a group runner") if runner.group_type?
+
         return if current_user.admin?
 
-        forbidden!("No access granted") unless user_can_access_runner?(runner)
+        forbidden!("Runner is locked") if runner.locked?
+        forbidden!("No access granted") unless can?(current_user, :assign_runner, runner)
       end
 
       def authenticate_list_runners_jobs!(runner)
         return if current_user.admin?
 
-        forbidden!("No access granted") unless user_can_access_runner?(runner)
-      end
-
-      def user_can_access_runner?(runner)
-        current_user.ci_authorized_runners.exists?(runner.id)
+        forbidden!("No access granted") unless can?(current_user, :read_runner, runner)
       end
     end
   end
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 152df23a327..e31c332b6e4 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -5,7 +5,7 @@ module API
     helpers do
       def current_settings
         @current_setting ||=
-          (ApplicationSetting.current || ApplicationSetting.create_from_defaults)
+          (ApplicationSetting.current_without_cache || ApplicationSetting.create_from_defaults)
       end
     end
 
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
index 2c52d21fa1c..4fa7d196e50 100644
--- a/lib/api/v3/groups.rb
+++ b/lib/api/v3/groups.rb
@@ -131,7 +131,9 @@ module API
         delete ":id" do
           group = find_group!(params[:id])
           authorize! :admin_group, group
-          present ::Groups::DestroyService.new(group, current_user).execute, with: Entities::GroupDetail, current_user: current_user
+          ::Groups::DestroyService.new(group, current_user).async_execute
+
+          accepted!
         end
 
         desc 'Get a list of projects in this group.' do
diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb
index c6d9957d452..8a5c46805bd 100644
--- a/lib/api/v3/runners.rb
+++ b/lib/api/v3/runners.rb
@@ -58,7 +58,7 @@ module API
         end
 
         def user_can_access_runner?(runner)
-          current_user.ci_authorized_runners.exists?(runner.id)
+          current_user.ci_owned_runners.exists?(runner.id)
         end
       end
     end
diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb
index 9b4ab7630fb..fc56495c8b1 100644
--- a/lib/api/v3/settings.rb
+++ b/lib/api/v3/settings.rb
@@ -6,7 +6,7 @@ module API
       helpers do
         def current_settings
           @current_setting ||=
-            (ApplicationSetting.current || ApplicationSetting.create_from_defaults)
+            (ApplicationSetting.current_without_cache || ApplicationSetting.create_from_defaults)
         end
       end
 
diff --git a/lib/api/version.rb b/lib/api/version.rb
index 9ba576bd828..3b10bfa6a7d 100644
--- a/lib/api/version.rb
+++ b/lib/api/version.rb
@@ -6,7 +6,7 @@ module API
       detail 'This feature was introduced in GitLab 8.13.'
     end
     get '/version' do
-      { version: Gitlab::VERSION, revision: Gitlab::REVISION }
+      { version: Gitlab::VERSION, revision: Gitlab.revision }
     end
   end
 end
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 6a5a223a614..45a935ab352 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Artifacts < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('artifacts', JobArtifactUploader.root)
     end
   end
diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index f869916e199..adf85ca4719 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Builds < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('builds', Settings.gitlab_ci.builds_path)
     end
   end
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 5e6828de597..1608f7ad02d 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -2,9 +2,11 @@ require 'yaml'
 
 module Backup
   class Database
+    attr_reader :progress
     attr_reader :config, :db_file_name
 
-    def initialize
+    def initialize(progress)
+      @progress = progress
       @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
@@ -19,12 +21,12 @@ module Backup
       dump_pid =
         case config["adapter"]
         when /^mysql/ then
-          $progress.print "Dumping MySQL database #{config['database']} ... "
+          progress.print "Dumping MySQL database #{config['database']} ... "
           # Workaround warnings from MySQL 5.6 about passwords on cmd line
           ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
           spawn('mysqldump', *mysql_args, config['database'], out: compress_wr)
         when "postgresql" then
-          $progress.print "Dumping PostgreSQL database #{config['database']} ... "
+          progress.print "Dumping PostgreSQL database #{config['database']} ... "
           pg_env
           pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
           if Gitlab.config.backup.pg_schema
@@ -53,12 +55,12 @@ module Backup
       restore_pid =
         case config["adapter"]
         when /^mysql/ then
-          $progress.print "Restoring MySQL database #{config['database']} ... "
+          progress.print "Restoring MySQL database #{config['database']} ... "
           # Workaround warnings from MySQL 5.6 about passwords on cmd line
           ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
           spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
         when "postgresql" then
-          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+          progress.print "Restoring PostgreSQL database #{config['database']} ... "
           pg_env
           spawn('psql', config['database'], in: decompress_rd)
         end
@@ -111,9 +113,9 @@ module Backup
 
     def report_success(success)
       if success
-        $progress.puts '[DONE]'.color(:green)
+        progress.puts '[DONE]'.color(:green)
       else
-        $progress.puts '[FAILED]'.color(:red)
+        progress.puts '[FAILED]'.color(:red)
       end
     end
   end
diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb
index 4e234e50a7a..185ff8ae6bd 100644
--- a/lib/backup/lfs.rb
+++ b/lib/backup/lfs.rb
@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Lfs < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('lfs', Settings.lfs.storage_path)
     end
   end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f27ce4d2b2b..a8da0c7edef 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -4,6 +4,12 @@ module Backup
     FOLDERS_TO_BACKUP = %w[repositories db].freeze
     FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze
 
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+    end
+
     def pack
       # Make sure there is a connection
       ActiveRecord::Base.connection.reconnect!
@@ -14,11 +20,11 @@ module Backup
         end
 
         # create archive
-        $progress.print "Creating backup archive: #{tar_file} ... "
+        progress.print "Creating backup archive: #{tar_file} ... "
         # Set file permissions on open to prevent chmod races.
         tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] }
         if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options)
-          $progress.puts "done".color(:green)
+          progress.puts "done".color(:green)
         else
           puts "creating archive #{tar_file} failed".color(:red)
           abort 'Backup failed'
@@ -29,11 +35,11 @@ module Backup
     end
 
     def upload
-      $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
+      progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
 
       connection_settings = Gitlab.config.backup.upload.connection
       if connection_settings.blank?
-        $progress.puts "skipped".color(:yellow)
+        progress.puts "skipped".color(:yellow)
         return
       end
 
@@ -43,7 +49,7 @@ module Backup
                                 multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
                                 encryption: Gitlab.config.backup.upload.encryption,
                                 storage_class: Gitlab.config.backup.upload.storage_class)
-        $progress.puts "done".color(:green)
+        progress.puts "done".color(:green)
       else
         puts "uploading backup to #{remote_directory} failed".color(:red)
         abort 'Backup failed'
@@ -51,13 +57,13 @@ module Backup
     end
 
     def cleanup
-      $progress.print "Deleting tmp directories ... "
+      progress.print "Deleting tmp directories ... "
 
       backup_contents.each do |dir|
         next unless File.exist?(File.join(backup_path, dir))
 
         if FileUtils.rm_rf(File.join(backup_path, dir))
-          $progress.puts "done".color(:green)
+          progress.puts "done".color(:green)
         else
           puts "deleting tmp directory '#{dir}' failed".color(:red)
           abort 'Backup failed'
@@ -67,7 +73,7 @@ module Backup
 
     def remove_old
       # delete backups
-      $progress.print "Deleting old backups ... "
+      progress.print "Deleting old backups ... "
       keep_time = Gitlab.config.backup.keep_time.to_i
 
       if keep_time > 0
@@ -88,31 +94,32 @@ module Backup
                 FileUtils.rm(file)
                 removed += 1
               rescue => e
-                $progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
+                progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
               end
             end
           end
         end
 
-        $progress.puts "done. (#{removed} removed)".color(:green)
+        progress.puts "done. (#{removed} removed)".color(:green)
       else
-        $progress.puts "skipping".color(:yellow)
+        progress.puts "skipping".color(:yellow)
       end
     end
 
+    # rubocop: disable Metrics/AbcSize
     def unpack
       Dir.chdir(backup_path) do
         # check for existing backups in the backup dir
         if backup_file_list.empty?
-          $progress.puts "No backups found in #{backup_path}"
-          $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
+          progress.puts "No backups found in #{backup_path}"
+          progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
           exit 1
         elsif backup_file_list.many? && ENV["BACKUP"].nil?
-          $progress.puts 'Found more than one backup:'
+          progress.puts 'Found more than one backup:'
           # print list of available backups
-          $progress.puts " " + available_timestamps.join("\n ")
-          $progress.puts 'Please specify which one you want to restore:'
-          $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
+          progress.puts " " + available_timestamps.join("\n ")
+          progress.puts 'Please specify which one you want to restore:'
+          progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
           exit 1
         end
 
@@ -123,31 +130,31 @@ module Backup
                    end
 
         unless File.exist?(tar_file)
-          $progress.puts "The backup file #{tar_file} does not exist!"
+          progress.puts "The backup file #{tar_file} does not exist!"
           exit 1
         end
 
-        $progress.print 'Unpacking backup ... '
+        progress.print 'Unpacking backup ... '
 
         unless Kernel.system(*%W(tar -xf #{tar_file}))
-          $progress.puts 'unpacking backup failed'.color(:red)
+          progress.puts 'unpacking backup failed'.color(:red)
           exit 1
         else
-          $progress.puts 'done'.color(:green)
+          progress.puts 'done'.color(:green)
         end
 
         ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
 
         # restoring mismatching backups can lead to unexpected problems
         if settings[:gitlab_version] != Gitlab::VERSION
-          $progress.puts(<<~HEREDOC.color(:red))
+          progress.puts(<<~HEREDOC.color(:red))
             GitLab version mismatch:
               Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!
               Please switch to the following version and try again:
               version: #{settings[:gitlab_version]}
           HEREDOC
-          $progress.puts
-          $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
+          progress.puts
+          progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
           exit 1
         end
       end
diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb
index 5830b209d6e..542e35a7c7c 100644
--- a/lib/backup/pages.rb
+++ b/lib/backup/pages.rb
@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Pages < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('pages', Gitlab.config.pages.path)
     end
   end
diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb
index 91698669402..35821805797 100644
--- a/lib/backup/registry.rb
+++ b/lib/backup/registry.rb
@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Registry < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('registry', Settings.registry.path)
     end
   end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 65e06fd78c0..c3360c391af 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -6,6 +6,12 @@ module Backup
     include Backup::Helper
     # rubocop:disable Metrics/AbcSize
 
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+    end
+
     def dump
       prepare
 
@@ -67,6 +73,9 @@ module Backup
     end
 
     def prepare_directories
+      # TODO: Need to find a way to do this for gitaly
+      # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194
+
       Gitlab.config.repositories.storages.each do |name, repository_storage|
         path = repository_storage.legacy_disk_path
         next unless File.exist?(path)
@@ -87,70 +96,65 @@ module Backup
       end
     end
 
+    def restore_custom_hooks(project)
+      # TODO: Need to find a way to do this for gitaly
+      # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
+      in_path(path_to_tars(project)) do |dir|
+        path_to_project_repo = path_to_repo(project)
+        cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
+
+        output, status = Gitlab::Popen.popen(cmd)
+        unless status.zero?
+          progress_warn(project, cmd.join(' '), output)
+        end
+      end
+    end
+
     def restore
       prepare_directories
+      gitlab_shell = Gitlab::Shell.new
       Project.find_each(batch_size: 1000) do |project|
-        progress.print " * #{display_repo_path(project)} ... "
-        path_to_project_repo = path_to_repo(project)
+        progress.print " * #{project.full_path} ... "
         path_to_project_bundle = path_to_bundle(project)
-
         project.ensure_storage_path_exists
 
-        cmd = if File.exist?(path_to_project_bundle)
-                %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo})
-              else
-                %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo})
-              end
+        restore_repo_success = nil
+        if File.exist?(path_to_project_bundle)
+          begin
+            gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists?
+            project.repository.create_from_bundle path_to_project_bundle
+            restore_repo_success = true
+          rescue => e
+            restore_repo_success = false
+            progress.puts "Error: #{e}".color(:red)
+          end
+        else
+          restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path)
+        end
 
-        output, status = Gitlab::Popen.popen(cmd)
-        if status.zero?
+        if restore_repo_success
           progress.puts "[DONE]".color(:green)
         else
-          progress_warn(project, cmd.join(' '), output)
+          progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
         end
 
-        in_path(path_to_tars(project)) do |dir|
-          cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-
-          output, status = Gitlab::Popen.popen(cmd)
-          unless status.zero?
-            progress_warn(project, cmd.join(' '), output)
-          end
-        end
+        restore_custom_hooks(project)
 
         wiki = ProjectWiki.new(project)
-        path_to_wiki_repo = path_to_repo(wiki)
         path_to_wiki_bundle = path_to_bundle(wiki)
 
         if File.exist?(path_to_wiki_bundle)
-          progress.print " * #{display_repo_path(wiki)} ... "
-
-          # If a wiki bundle exists, first remove the empty repo
-          # that was initialized with ProjectWiki.new() and then
-          # try to restore with 'git clone --bare'.
-          FileUtils.rm_rf(path_to_wiki_repo)
-          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo})
-
-          output, status = Gitlab::Popen.popen(cmd)
-          if status.zero?
-            progress.puts " [DONE]".color(:green)
-          else
-            progress_warn(project, cmd.join(' '), output)
+          progress.print " * #{wiki.full_path} ... "
+          begin
+            gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists?
+            wiki.repository.create_from_bundle(path_to_wiki_bundle)
+            progress.puts "[DONE]".color(:green)
+          rescue => e
+            progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
+            progress.puts "Error #{e}".color(:red)
           end
         end
       end
-
-      progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
-      cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
-
-      output, status = Gitlab::Popen.popen(cmd)
-      if status.zero?
-        progress.puts " [DONE]".color(:green)
-      else
-        puts " [FAILED]".color(:red)
-        puts "failed: #{cmd}"
-        puts output
-      end
     end
     # rubocop:enable Metrics/AbcSize
 
@@ -217,10 +221,6 @@ module Backup
       Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
     end
 
-    def progress
-      $progress
-    end
-
     def display_repo_path(project)
       project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
     end
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index d46e2cd869d..49b117a7ee3 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -2,7 +2,11 @@ require 'backup/files'
 
 module Backup
   class Uploads < Files
-    def initialize
+    attr_reader :progress
+
+    def initialize(progress)
+      @progress = progress
+
       super('uploads', Rails.root.join('public/uploads'))
     end
   end
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
index f2e9a5a1116..4bc82ecb4d6 100644
--- a/lib/banzai/filter/gollum_tags_filter.rb
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -58,6 +58,9 @@ module Banzai
 
       def call
         doc.search(".//text()").each do |node|
+          # Do not perform linking inside <code> blocks
+          next unless node.ancestors('code').empty?
+
           # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running
           # before this one, it will be converted into `[[<em>TOC</em>]]`, so it
           # needs special-case handling
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
index 5325819d828..28933c78966 100644
--- a/lib/banzai/filter/plantuml_filter.rb
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -23,7 +23,7 @@ module Banzai
       private
 
       def settings
-        ApplicationSetting.current || ApplicationSetting.create_from_defaults
+        Gitlab::CurrentSettings.current_application_settings
       end
 
       def plantuml_setup
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index b9d5ecf70ec..2f023f4f242 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -73,7 +73,7 @@ module Banzai
       #
       # Note that while the key might exist, its value could be nil!
       def validate
-        needs :project
+        needs :project unless skip_project_check?
       end
 
       # Iterates over all <a> and text() nodes in a document.
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 8b2f05fffec..a1f24e8b093 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -42,9 +42,9 @@ module Banzai
       end
 
       def self.transform_context(context)
-        context.merge(
-          only_path: true,
+        context[:only_path] = true unless context.key?(:only_path)
 
+        context.merge(
           # EmojiFilter
           asset_host: Gitlab::Application.config.asset_host,
           asset_root: Gitlab.config.gitlab.base_url
diff --git a/lib/feature.rb b/lib/feature.rb
index 8e9ba5c530a..6474de6e56d 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -1,3 +1,6 @@
+require 'flipper/adapters/active_record'
+require 'flipper/adapters/active_support_cache_store'
+
 class Feature
   # Classes to override flipper table names
   class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
@@ -60,7 +63,8 @@ class Feature
     end
 
     def flipper
-      @flipper ||= Flipper.instance
+      Thread.current[:flipper] ||=
+        Flipper.new(flipper_adapter).tap { |flip| flip.memoize = true }
     end
 
     # This method is called from config/initializers/flipper.rb and can be used
@@ -68,5 +72,16 @@ class Feature
     # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups
     def register_feature_groups
     end
+
+    def flipper_adapter
+      active_record_adapter = Flipper::Adapters::ActiveRecord.new(
+        feature_class: FlipperFeature,
+        gate_class: FlipperGate)
+
+      Flipper::Adapters::ActiveSupportCacheStore.new(
+        active_record_adapter,
+        Rails.cache,
+        expires_in: 1.hour)
+    end
   end
 end
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index c5498d0da1a..a129746e2c6 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -9,11 +9,30 @@ module Gitlab
     Settings
   end
 
+  def self.migrations_hash
+    @_migrations_hash ||= Digest::MD5.hexdigest(ActiveRecord::Migrator.get_all_versions.to_s)
+  end
+
+  def self.revision
+    @_revision ||= begin
+      if File.exist?(root.join("REVISION"))
+        File.read(root.join("REVISION")).strip.freeze
+      else
+        result = Gitlab::Popen.popen_with_detail(%W[#{config.git.bin_path} log --pretty=format:%h -n 1])
+
+        if result.status.success?
+          result.stdout.chomp.freeze
+        else
+          "Unknown".freeze
+        end
+      end
+    end
+  end
+
   COM_URL = 'https://gitlab.com'.freeze
   APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}
   SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}
   VERSION = File.read(root.join("VERSION")).strip.freeze
-  REVISION = Gitlab::Popen.popen(%W(#{config.git.bin_path} log --pretty=format:%h -n 1)).first.chomp.freeze
 
   def self.com?
     # Check `gl_subdomain?` as well to keep parity with gitlab.com
diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb
index dae03a179e4..7609a7b04f6 100644
--- a/lib/gitlab/auth/blocked_user_tracker.rb
+++ b/lib/gitlab/auth/blocked_user_tracker.rb
@@ -17,7 +17,9 @@ module Gitlab
         # message passed along by Warden.
         return unless message == User::BLOCKED_MESSAGE
 
-        login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login')
+        # Check for either LDAP or regular GitLab account logins
+        login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') ||
+          env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login')
 
         return unless login.present?
 
diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb
index 34286900e72..865185eb5db 100644
--- a/lib/gitlab/auth/ldap/access.rb
+++ b/lib/gitlab/auth/ldap/access.rb
@@ -6,7 +6,7 @@ module Gitlab
   module Auth
     module LDAP
       class Access
-        attr_reader :provider, :user
+        attr_reader :provider, :user, :ldap_identity
 
         def self.open(user, &block)
           Gitlab::Auth::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter|
@@ -14,9 +14,12 @@ module Gitlab
           end
         end
 
-        def self.allowed?(user)
+        def self.allowed?(user, options = {})
           self.open(user) do |access|
+            # Whether user is allowed, or not, we should update
+            # permissions to keep things clean
             if access.allowed?
+              access.update_user
               Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute
 
               true
@@ -29,7 +32,8 @@ module Gitlab
         def initialize(user, adapter = nil)
           @adapter = adapter
           @user = user
-          @provider = user.ldap_identity.provider
+          @ldap_identity = user.ldap_identity
+          @provider = adapter&.provider || ldap_identity&.provider
         end
 
         def allowed?
@@ -40,7 +44,7 @@ module Gitlab
             end
 
             # Block user in GitLab if he/she was blocked in AD
-            if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter)
+            if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(ldap_identity.extern_uid, adapter)
               block_user(user, 'is disabled in Active Directory')
               false
             else
@@ -64,27 +68,44 @@ module Gitlab
           Gitlab::Auth::LDAP::Config.new(provider)
         end
 
+        def find_ldap_user
+          Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter)
+        end
+
         def ldap_user
-          @ldap_user ||= Gitlab::Auth::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter)
+          return unless provider
+
+          @ldap_user ||= find_ldap_user
         end
 
         def block_user(user, reason)
           user.ldap_block
 
-          Gitlab::AppLogger.info(
-            "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \
-            "blocking Gitlab user \"#{user.name}\" (#{user.email})"
-          )
+          if provider
+            Gitlab::AppLogger.info(
+              "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \
+              "blocking Gitlab user \"#{user.name}\" (#{user.email})"
+            )
+          else
+            Gitlab::AppLogger.info(
+              "Account is not provided by LDAP, " \
+              "blocking Gitlab user \"#{user.name}\" (#{user.email})"
+            )
+          end
         end
 
         def unblock_user(user, reason)
           user.activate
 
           Gitlab::AppLogger.info(
-            "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \
+            "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \
             "unblocking Gitlab user \"#{user.name}\" (#{user.email})"
           )
         end
+
+        def update_user
+          # no-op in CE
+        end
       end
     end
   end
diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb
index 77185f52ced..d4415eaa6dc 100644
--- a/lib/gitlab/auth/ldap/config.rb
+++ b/lib/gitlab/auth/ldap/config.rb
@@ -11,6 +11,8 @@ module Gitlab
 
         attr_accessor :provider, :options
 
+        InvalidProvider = Class.new(StandardError)
+
         def self.enabled?
           Gitlab.config.ldap.enabled
         end
@@ -22,6 +24,10 @@ module Gitlab
         def self.available_servers
           return [] unless enabled?
 
+          _available_servers
+        end
+
+        def self._available_servers
           Array.wrap(servers.first)
         end
 
@@ -34,7 +40,7 @@ module Gitlab
         end
 
         def self.invalid_provider(provider)
-          raise "Unknown provider (#{provider}). Available providers: #{providers}"
+          raise InvalidProvider.new("Unknown provider (#{provider}). Available providers: #{providers}")
         end
 
         def initialize(provider)
@@ -84,13 +90,17 @@ module Gitlab
         end
 
         def base
-          options['base']
+          @base ||= Person.normalize_dn(options['base'])
         end
 
         def uid
           options['uid']
         end
 
+        def label
+          options['label']
+        end
+
         def sync_ssh_keys?
           sync_ssh_keys.present?
         end
@@ -132,6 +142,10 @@ module Gitlab
           options['timeout'].to_i
         end
 
+        def external_groups
+          options['external_groups'] || []
+        end
+
         def has_auth?
           options['password'] || options['bind_dn']
         end
diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb
index 2760b1a3247..5fa9581f837 100644
--- a/lib/gitlab/auth/saml/config.rb
+++ b/lib/gitlab/auth/saml/config.rb
@@ -14,6 +14,10 @@ module Gitlab
           def external_groups
             options[:external_groups]
           end
+
+          def admin_groups
+            options[:admin_groups]
+          end
         end
       end
     end
diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb
index cb01cd8004c..b8c84c37cd5 100644
--- a/lib/gitlab/auth/saml/user.rb
+++ b/lib/gitlab/auth/saml/user.rb
@@ -20,10 +20,8 @@ module Gitlab
           user ||= find_or_build_ldap_user if auto_link_ldap_user?
           user ||= build_new_user if signup_enabled?
 
-          if external_users_enabled? && user
-            # Check if there is overlap between the user's groups and the external groups
-            # setting then set user as external or internal.
-            user.external = !(auth_hash.groups & saml_config.external_groups).empty?
+          if user
+            user.external = !(auth_hash.groups & saml_config.external_groups).empty? if external_users_enabled?
           end
 
           user
diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb
index cf02030c577..4dc23f977da 100644
--- a/lib/gitlab/auth/user_auth_finders.rb
+++ b/lib/gitlab/auth/user_auth_finders.rb
@@ -1,9 +1,5 @@
 module Gitlab
   module Auth
-    #
-    # Exceptions
-    #
-
     AuthenticationError = Class.new(StandardError)
     MissingTokenError = Class.new(AuthenticationError)
     TokenNotFoundError = Class.new(AuthenticationError)
@@ -61,6 +57,12 @@ module Gitlab
 
       private
 
+      def route_authentication_setting
+        return {} unless respond_to?(:route_setting)
+
+        route_setting(:authentication) || {}
+      end
+
       def access_token
         strong_memoize(:access_token) do
           find_oauth_access_token || find_personal_access_token
diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb
new file mode 100644
index 00000000000..f57df7c5637
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression.rb
@@ -0,0 +1,10 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Expression
+        ExpressionError = Class.new(StandardError)
+        RuntimeError = Class.new(ExpressionError)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
new file mode 100644
index 00000000000..10957598f76
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Expression
+        module Lexeme
+          class Matches < Lexeme::Operator
+            PATTERN = /=~/.freeze
+
+            def initialize(left, right)
+              @left = left
+              @right = right
+            end
+
+            def evaluate(variables = {})
+              text = @left.evaluate(variables)
+              regexp = @right.evaluate(variables)
+
+              regexp.scan(text.to_s).any?
+            end
+
+            def self.build(_value, behind, ahead)
+              new(behind, ahead)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
new file mode 100644
index 00000000000..9b239c29ea4
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
@@ -0,0 +1,33 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Expression
+        module Lexeme
+          require_dependency 're2'
+
+          class Pattern < Lexeme::Value
+            PATTERN = %r{^/.+/[ismU]*$}.freeze
+
+            def initialize(regexp)
+              @value = regexp
+
+              unless Gitlab::UntrustedRegexp.valid?(@value)
+                raise Lexer::SyntaxError, 'Invalid regular expression!'
+              end
+            end
+
+            def evaluate(variables = {})
+              Gitlab::UntrustedRegexp.fabricate(@value)
+            rescue RegexpError
+              raise Expression::RuntimeError, 'Invalid regular expression!'
+            end
+
+            def self.build(string)
+              new(string)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb
index e1c68b7c3c2..4cacb1e62c9 100644
--- a/lib/gitlab/ci/pipeline/expression/lexer.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexer.rb
@@ -5,15 +5,17 @@ module Gitlab
         class Lexer
           include ::Gitlab::Utils::StrongMemoize
 
+          SyntaxError = Class.new(Expression::ExpressionError)
+
           LEXEMES = [
             Expression::Lexeme::Variable,
             Expression::Lexeme::String,
+            Expression::Lexeme::Pattern,
             Expression::Lexeme::Null,
-            Expression::Lexeme::Equals
+            Expression::Lexeme::Equals,
+            Expression::Lexeme::Matches
           ].freeze
 
-          SyntaxError = Class.new(Statement::StatementError)
-
           MAX_TOKENS = 100
 
           def initialize(statement, max_tokens: MAX_TOKENS)
diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb
index 09a7c98464b..b36f1e0f865 100644
--- a/lib/gitlab/ci/pipeline/expression/statement.rb
+++ b/lib/gitlab/ci/pipeline/expression/statement.rb
@@ -3,15 +3,16 @@ module Gitlab
     module Pipeline
       module Expression
         class Statement
-          StatementError = Class.new(StandardError)
+          StatementError = Class.new(Expression::ExpressionError)
 
           GRAMMAR = [
+            %w[variable],
             %w[variable equals string],
             %w[variable equals variable],
             %w[variable equals null],
             %w[string equals variable],
             %w[null equals variable],
-            %w[variable]
+            %w[variable matches pattern]
           ].freeze
 
           def initialize(statement, variables = {})
@@ -35,11 +36,13 @@ module Gitlab
 
           def truthful?
             evaluate.present?
+          rescue Expression::ExpressionError
+            false
           end
 
           def valid?
             parse_tree.is_a?(Lexeme::Base)
-          rescue StatementError
+          rescue Expression::ExpressionError
             false
           end
         end
diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb
new file mode 100644
index 00000000000..e7a2e5511cf
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/preloader.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      # Class for preloading data associated with pipelines such as commit
+      # authors.
+      module Preloader
+        def self.preload(pipelines)
+          # This ensures that all the pipeline commits are eager loaded before we
+          # start using them.
+          pipelines.each(&:commit)
+
+          pipelines.each do |pipeline|
+            # This preloads the author of every commit. We're using "lazy_author"
+            # here since "author" immediately loads the data on the first call.
+            pipeline.commit.try(:lazy_author)
+
+            # This preloads the number of warnings for every pipeline, ensuring
+            # that Ci::Pipeline#has_warnings? doesn't execute any additional
+            # queries.
+            pipeline.number_of_warnings
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index e392a015b91..6cf7aa1bf0d 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -9,8 +9,8 @@ module Gitlab
         end
       end
 
-      def fake_application_settings(defaults = ::ApplicationSetting.defaults)
-        Gitlab::FakeApplicationSettings.new(defaults)
+      def fake_application_settings(attributes = {})
+        Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {}))
       end
 
       def method_missing(name, *args, &block)
@@ -25,43 +25,35 @@ module Gitlab
 
       def ensure_application_settings!
         return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true'
-
-        cached_application_settings || uncached_application_settings
-      end
-
-      def cached_application_settings
-        begin
-          ::ApplicationSetting.cached
-        rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL
-          # In case Redis isn't running or the Redis UNIX socket file is not available
-        end
-      end
-
-      def uncached_application_settings
         return fake_application_settings unless connect_to_db?
 
-        db_settings = ::ApplicationSetting.current
-
+        current_settings = ::ApplicationSetting.current
         # If there are pending migrations, it's possible there are columns that
         # need to be added to the application settings. To prevent Rake tasks
         # and other callers from failing, use any loaded settings and return
         # defaults for missing columns.
         if ActiveRecord::Migrator.needs_migration?
-          defaults = ::ApplicationSetting.defaults
-          defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present?
-          return fake_application_settings(defaults)
+          return fake_application_settings(current_settings&.attributes)
         end
 
-        return db_settings if db_settings.present?
+        return current_settings if current_settings.present?
 
-        ::ApplicationSetting.create_from_defaults || in_memory_application_settings
+        with_fallback_to_fake_application_settings do
+          ::ApplicationSetting.create_from_defaults || in_memory_application_settings
+        end
       end
 
       def in_memory_application_settings
-        @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables
-      rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError
-        # In case migrations the application_settings table is not created yet,
-        # we fallback to a simple OpenStruct
+        with_fallback_to_fake_application_settings do
+          @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults # rubocop:disable Gitlab/ModuleWithInstanceVariables
+        end
+      end
+
+      def with_fallback_to_fake_application_settings(&block)
+        yield
+      rescue
+        # In case the application_settings table is not created yet, or if a new
+        # ApplicationSetting column is not yet migrated we fallback to a simple OpenStruct
         fake_application_settings
       end
 
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 76501dd50e8..d49d055c3f2 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -43,7 +43,7 @@ module Gitlab
     end
 
     def self.version
-      database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
+      @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
     end
 
     def self.join_lateral_supported?
diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb
new file mode 100644
index 00000000000..3374203960e
--- /dev/null
+++ b/lib/gitlab/database/count.rb
@@ -0,0 +1,48 @@
+# For large tables, PostgreSQL can take a long time to count rows due to MVCC.
+# We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting.
+module Gitlab
+  module Database
+    module Count
+      CONNECTION_ERRORS =
+        if defined?(PG)
+          [
+            ActionView::Template::Error,
+            ActiveRecord::StatementInvalid,
+            PG::Error
+          ].freeze
+        else
+          [
+            ActionView::Template::Error,
+            ActiveRecord::StatementInvalid
+          ].freeze
+        end
+
+      def self.approximate_count(model)
+        return model.count unless Gitlab::Database.postgresql?
+
+        execute_estimate_if_updated_recently(model) || model.count
+      end
+
+      def self.execute_estimate_if_updated_recently(model)
+        ActiveRecord::Base.connection.select_value(postgresql_estimate_query(model)).to_i if reltuples_updated_recently?(model)
+      rescue *CONNECTION_ERRORS
+      end
+
+      def self.reltuples_updated_recently?(model)
+        time = "to_timestamp(#{1.hour.ago.to_i})"
+        query = <<~SQL
+          SELECT 1 FROM pg_stat_user_tables WHERE relname = '#{model.table_name}' AND
+          (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time})
+        SQL
+
+        ActiveRecord::Base.connection.select_all(query).count > 0
+      rescue *CONNECTION_ERRORS
+        false
+      end
+
+      def self.postgresql_estimate_query(model)
+        "SELECT reltuples::bigint AS estimate FROM pg_class where relname = '#{model.table_name}'"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb
index 05a60deb7d3..764f93f6d3d 100644
--- a/lib/gitlab/email/handler/create_issue_handler.rb
+++ b/lib/gitlab/email/handler/create_issue_handler.rb
@@ -47,7 +47,7 @@ module Gitlab
             project,
             author,
             title:       mail.subject,
-            description: message
+            description: message_including_reply
           ).execute
         end
       end
diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb
index da5ff350549..38b1425364f 100644
--- a/lib/gitlab/email/handler/reply_processing.rb
+++ b/lib/gitlab/email/handler/reply_processing.rb
@@ -16,8 +16,12 @@ module Gitlab
           @message ||= process_message
         end
 
-        def process_message
-          message = ReplyParser.new(mail).execute.strip
+        def message_including_reply
+          @message_with_reply ||= process_message(trim_reply: false)
+        end
+
+        def process_message(**kwargs)
+          message = ReplyParser.new(mail, **kwargs).execute.strip
           add_attachments(message)
         end
 
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 01c28d051ee..ae6b84607d6 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -4,8 +4,9 @@ module Gitlab
     class ReplyParser
       attr_accessor :message
 
-      def initialize(message)
+      def initialize(message, trim_reply: true)
         @message = message
+        @trim_reply = trim_reply
       end
 
       def execute
@@ -13,7 +14,9 @@ module Gitlab
 
         encoding = body.encoding
 
-        body = EmailReplyTrimmer.trim(body)
+        if @trim_reply
+          body = EmailReplyTrimmer.trim(body)
+        end
 
         return '' unless body
 
diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb
index 77af7a868d0..49bc9c0b671 100644
--- a/lib/gitlab/file_detector.rb
+++ b/lib/gitlab/file_detector.rb
@@ -14,7 +14,7 @@ module Gitlab
       avatar: /\Alogo\.(png|jpg|gif)\z/,
       issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z},
       merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z},
-      xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)\z},
+      xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)(/.+)?\z},
 
       # Configuration files
       gitignore: '.gitignore',
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index d78d29b7ac6..156d077a69c 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -104,25 +104,22 @@ module Gitlab
         #       file.rb      # oid: 4a
         #
         #
-        # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a'
+        # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a'
         #
-        def find_entry_by_path(repository, root_id, path)
+        def find_entry_by_path(repository, root_id, *path_parts)
           root_tree = repository.lookup(root_id)
-          # Strip leading slashes
-          path[%r{^/*}] = ''
-          path_arr = path.split('/')
 
           entry = root_tree.find do |entry|
-            entry[:name] == path_arr[0]
+            entry[:name] == path_parts[0]
           end
 
           return nil unless entry
 
-          if path_arr.size > 1
+          if path_parts.size > 1
             return nil unless entry[:type] == :tree
 
-            path_arr.shift
-            find_entry_by_path(repository, entry[:oid], path_arr.join('/'))
+            path_parts.shift
+            find_entry_by_path(repository, entry[:oid], *path_parts)
           else
             [:blob, :commit].include?(entry[:type]) ? entry : nil
           end
@@ -185,10 +182,13 @@ module Gitlab
         def find_by_rugged(repository, sha, path, limit:)
           return unless path
 
+          # Strip any leading / characters from the path
+          path = path.sub(%r{\A/*}, '')
+
           rugged_commit = repository.lookup(sha)
           root_tree = rugged_commit.tree
 
-          blob_entry = find_entry_by_path(repository, root_tree.oid, path)
+          blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/'))
 
           return nil unless blob_entry
 
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index fabcd46c8e9..89f761dd515 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -6,6 +6,7 @@ module Gitlab
 
       attr_accessor :raw_commit, :head
 
+      MAX_COMMIT_MESSAGE_DISPLAY_SIZE = 10.megabytes
       MIN_SHA_LENGTH = 7
       SERIALIZE_KEYS = [
         :id, :message, :parent_ids,
@@ -63,9 +64,7 @@ module Gitlab
             if is_enabled
               repo.gitaly_commit_client.find_commit(commit_id)
             else
-              obj = repo.rev_parse_target(commit_id)
-
-              obj.is_a?(Rugged::Commit) ? obj : nil
+              rugged_find(repo, commit_id)
             end
           end
 
@@ -76,6 +75,12 @@ module Gitlab
           nil
         end
 
+        def rugged_find(repo, commit_id)
+          obj = repo.rev_parse_target(commit_id)
+
+          obj.is_a?(Rugged::Commit) ? obj : nil
+        end
+
         # Get last commit for HEAD
         #
         # Ex.
@@ -297,11 +302,40 @@ module Gitlab
             nil
           end
         end
+
+        def get_message(repository, commit_id)
+          BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader|
+            items_by_repo = items.group_by { |i| i[:repository] }
+
+            items_by_repo.each do |repo, items|
+              commit_ids = items.map { |i| i[:commit_id] }
+
+              messages = get_messages(repository, commit_ids)
+
+              messages.each do |commit_sha, message|
+                loader.call({ repository: repository, commit_id: commit_sha }, message)
+              end
+            end
+          end
+        end
+
+        def get_messages(repository, commit_ids)
+          repository.gitaly_migrate(:commit_messages) do |is_enabled|
+            if is_enabled
+              repository.gitaly_commit_client.get_commit_messages(commit_ids)
+            else
+              commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h
+            end
+          end
+        end
       end
 
       def initialize(repository, raw_commit, head = nil)
         raise "Nil as raw commit passed" unless raw_commit
 
+        @repository = repository
+        @head = head
+
         case raw_commit
         when Hash
           init_from_hash(raw_commit)
@@ -312,9 +346,6 @@ module Gitlab
         else
           raise "Invalid raw commit type: #{raw_commit.class}"
         end
-
-        @repository = repository
-        @head = head
       end
 
       def sha
@@ -342,21 +373,6 @@ module Gitlab
         parent_ids.first
       end
 
-      # Shows the diff between the commit's parent and the commit.
-      #
-      # Cuts out the header and stats from #to_patch and returns only the diff.
-      #
-      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324
-      def to_diff
-        Gitlab::GitalyClient.migrate(:commit_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            @repository.gitaly_commit_client.patch(id)
-          else
-            rugged_diff_from_parent.patch
-          end
-        end
-      end
-
       # Returns a diff object for the changes from this commit's first parent.
       # If there is no parent, then the diff is between this commit and an
       # empty repo. See Repository#diff for keys allowed in the +options+
@@ -432,16 +448,6 @@ module Gitlab
         Gitlab::Git::CommitStats.new(@repository, self)
       end
 
-      def to_patch(options = {})
-        begin
-          rugged_commit.to_mbox(options)
-        rescue Rugged::InvalidError => ex
-          if ex.message =~ /commit \w+ is a merge commit/i
-            'Patch format is not currently supported for merge commits.'
-          end
-        end
-      end
-
       # Get ref names collection
       #
       # Ex.
@@ -543,7 +549,7 @@ module Gitlab
         # TODO: Once gitaly "takes over" Rugged consider separating the
         # subject from the message to make it clearer when there's one
         # available but not the other.
-        @message = (commit.body.presence || commit.subject).dup
+        @message = message_from_gitaly_body
         @authored_date = Time.at(commit.author.date.seconds).utc
         @author_name = commit.author.name.dup
         @author_email = commit.author.email.dup
@@ -595,6 +601,25 @@ module Gitlab
       def refs(repo)
         repo.refs_hash[id]
       end
+
+      def message_from_gitaly_body
+        return @raw_commit.subject.dup if @raw_commit.body_size.zero?
+        return @raw_commit.body.dup if full_body_fetched_from_gitaly?
+
+        if @raw_commit.body_size > MAX_COMMIT_MESSAGE_DISPLAY_SIZE
+          "#{@raw_commit.subject}\n\n--commit message is too big".strip
+        else
+          fetch_body_from_gitaly
+        end
+      end
+
+      def full_body_fetched_from_gitaly?
+        @raw_commit.body.bytesize == @raw_commit.body_size
+      end
+
+      def fetch_body_from_gitaly
+        self.class.get_message(@repository, id)
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb
index 68373460d23..00c943fdb25 100644
--- a/lib/gitlab/git/gitlab_projects.rb
+++ b/lib/gitlab/git/gitlab_projects.rb
@@ -53,7 +53,7 @@ module Gitlab
       # Import project via git clone --bare
       # URL must be publicly cloneable
       def import_project(source, timeout)
-        Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled|
+        Gitlab::GitalyClient.migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_import_repository(source)
           else
diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb
index 155cf52f050..57b82a37d6c 100644
--- a/lib/gitlab/git/path_helper.rb
+++ b/lib/gitlab/git/path_helper.rb
@@ -6,7 +6,7 @@ module Gitlab
       class << self
         def normalize_path(filename)
           # Strip all leading slashes so that //foo -> foo
-          filename[%r{^/*}] = ''
+          filename = filename.sub(%r{\A/*}, '')
 
           # Expand relative paths (e.g. foo/../bar)
           filename = Pathname.new(filename)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 5d47f8b2075..1a21625a322 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -403,10 +403,10 @@ module Gitlab
         prefix = archive_prefix(ref, commit.id, append_sha: append_sha)
 
         {
-          'RepoPath' => path,
           'ArchivePrefix' => prefix,
           'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format),
-          'CommitId' => commit.id
+          'CommitId' => commit.id,
+          'GitalyRepository' => gitaly_repository.to_h
         }
       end
 
@@ -579,11 +579,6 @@ module Gitlab
         count_commits(from: from, to: to, **options)
       end
 
-      # Counts the amount of commits between `from` and `to`.
-      def count_commits_between(from, to, options = {})
-        count_commits(from: from, to: to, **options)
-      end
-
       # old_rev and new_rev are commit ID's
       # the result of this method is an array of Gitlab::Git::RawDiffChange
       def raw_changes_between(old_rev, new_rev)
@@ -781,13 +776,9 @@ module Gitlab
       end
 
       def add_branch(branch_name, user:, target:)
-        gitaly_migrate(:operation_user_create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_add_branch(branch_name, user, target)
-          else
-            rugged_add_branch(branch_name, user, target)
-          end
-        end
+        gitaly_operation_client.user_create_branch(branch_name, user, target)
+      rescue GRPC::FailedPrecondition => ex
+        raise InvalidRef, ex
       end
 
       def add_tag(tag_name, user:, target:, message: nil)
@@ -1057,7 +1048,7 @@ module Gitlab
         return @info_attributes if @info_attributes
 
         content =
-          gitaly_migrate(:get_info_attributes) do |is_enabled|
+          gitaly_migrate(:get_info_attributes, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
             if is_enabled
               gitaly_repository_client.info_attributes
             else
@@ -1343,7 +1334,7 @@ module Gitlab
       end
 
       def squash_in_progress?(squash_id)
-        gitaly_migrate(:squash_in_progress) do |is_enabled|
+        gitaly_migrate(:squash_in_progress, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_repository_client.squash_in_progress?(squash_id)
           else
@@ -1472,34 +1463,29 @@ module Gitlab
       end
 
       def branch_names_contains_sha(sha)
-        gitaly_migrate(:branch_names_contains_sha,
-                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.branch_names_contains_sha(sha)
-          else
-            refs_contains_sha('refs/heads/', sha)
-          end
-        end
+        gitaly_ref_client.branch_names_contains_sha(sha)
       end
 
       def tag_names_contains_sha(sha)
-        gitaly_migrate(:tag_names_contains_sha,
-                       status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_ref_client.tag_names_contains_sha(sha)
-          else
-            refs_contains_sha('refs/tags/', sha)
-          end
-        end
+        gitaly_ref_client.tag_names_contains_sha(sha)
       end
 
       def search_files_by_content(query, ref)
         return [] if empty? || query.blank?
 
-        offset = 2
-        args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
+        safe_query = Regexp.escape(query)
+        ref ||= root_ref
 
-        run_git(args).first.scrub.split(/^--\n/)
+        gitaly_migrate(:search_files_by_content) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.search_files_by_content(ref, safe_query)
+          else
+            offset = 2
+            args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref})
+
+            run_git(args).first.scrub.split(/^--\n/)
+          end
+        end
       end
 
       def can_be_merged?(source_sha, target_branch)
@@ -1514,12 +1500,19 @@ module Gitlab
 
       def search_files_by_name(query, ref)
         safe_query = Regexp.escape(query.sub(%r{^/*}, ""))
+        ref ||= root_ref
 
         return [] if empty? || safe_query.blank?
 
-        args = %W(ls-tree -r --name-status --full-tree #{ref || root_ref} -- #{safe_query})
+        gitaly_migrate(:search_files_by_name) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.search_files_by_name(ref, safe_query)
+          else
+            args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query})
 
-        run_git(args).first.lines.map(&:strip)
+            run_git(args).first.lines.map(&:strip)
+          end
+        end
       end
 
       def find_commits_by_message(query, ref, path, limit, offset)
@@ -1595,14 +1588,12 @@ module Gitlab
       end
 
       def checksum
-        gitaly_migrate(:calculate_checksum,
-                       status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_repository_client.calculate_checksum
-          else
-            calculate_checksum_by_shelling_out
-          end
-        end
+        # The exists? RPC is much cheaper, so we perform this request first
+        raise NoRepository, "Repository does not exists" unless exists?
+
+        gitaly_repository_client.calculate_checksum
+      rescue GRPC::NotFound
+        raise NoRepository # Guard against data races.
       end
 
       private
@@ -1625,27 +1616,6 @@ module Gitlab
         end
       end
 
-      def refs_contains_sha(refs_prefix, sha)
-        refs_prefix << "/" unless refs_prefix.ends_with?('/')
-
-        # By forcing the output to %(refname) each line wiht a ref will start with
-        # the ref prefix. All other lines can be discarded.
-        args = %W(for-each-ref --contains=#{sha} --format=%(refname) #{refs_prefix})
-        names, code = run_git(args)
-
-        return [] unless code.zero?
-
-        refs = []
-        left_slice_count = refs_prefix.length
-        names.lines.each do |line|
-          next unless line.start_with?(refs_prefix)
-
-          refs << encode_utf8(line.rstrip[left_slice_count..-1])
-        end
-
-        refs
-      end
-
       def rugged_write_config(full_path:)
         rugged.config['gitlab.fullpath'] = full_path
       end
@@ -1992,7 +1962,12 @@ module Gitlab
           end
 
           target_commit = Gitlab::Git::Commit.find(self, ref.target)
-          Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message)
+          Gitlab::Git::Tag.new(self, {
+            name: ref.name,
+            target: ref.target,
+            target_commit: target_commit,
+            message: message
+          })
         end.sort_by(&:name)
       end
 
@@ -2237,22 +2212,6 @@ module Gitlab
         end
       end
 
-      def gitaly_add_branch(branch_name, user, target)
-        gitaly_operation_client.user_create_branch(branch_name, user, target)
-      rescue GRPC::FailedPrecondition => ex
-        raise InvalidRef, ex
-      end
-
-      def rugged_add_branch(branch_name, user, target)
-        target_object = Ref.dereference_object(lookup(target))
-        raise InvalidRef.new("target not found: #{target}") unless target_object
-
-        OperationService.new(user, self).add_branch(branch_name, target_object.oid)
-        find_branch(branch_name)
-      rescue Rugged::ReferenceError => ex
-        raise InvalidRef, ex
-      end
-
       def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
         OperationService.new(user, self).with_branch(
           branch_name,
@@ -2408,7 +2367,7 @@ module Gitlab
       end
 
       def gitaly_delete_refs(*ref_names)
-        gitaly_ref_client.delete_refs(refs: ref_names)
+        gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any?
       end
 
       def rugged_remove_remote(remote_name)
@@ -2538,36 +2497,6 @@ module Gitlab
         rev_parse_target(ref).oid
       end
 
-      def calculate_checksum_by_shelling_out
-        raise NoRepository unless exists?
-
-        args = %W(--git-dir=#{path} show-ref --heads --tags)
-        output, status = run_git(args)
-
-        if status.nil? || !status.zero?
-          # Non-valid git repositories return 128 as the status code and an error output
-          raise InvalidRepository if status == 128 && output.to_s.downcase =~ /not a git repository/
-          # Empty repositories returns with a non-zero status and an empty output.
-          raise ChecksumError, output unless output.blank?
-
-          return EMPTY_REPOSITORY_CHECKSUM
-        end
-
-        refs = output.split("\n")
-
-        result = refs.inject(nil) do |checksum, ref|
-          value = Digest::SHA1.hexdigest(ref).hex
-
-          if checksum.nil?
-            value
-          else
-            checksum ^ value
-          end
-        end
-
-        result.to_s(16)
-      end
-
       def build_git_cmd(*args)
         object_directories = alternate_object_directories.join(File::PATH_SEPARATOR)
 
diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb
index 8a01f92e2af..e35ea5762eb 100644
--- a/lib/gitlab/git/repository_mirroring.rb
+++ b/lib/gitlab/git/repository_mirroring.rb
@@ -35,7 +35,11 @@ module Gitlab
           next if name =~ /\^\{\}\Z/
 
           target_commit = Gitlab::Git::Commit.find(self, target)
-          Gitlab::Git::Tag.new(self, name, target, target_commit)
+          Gitlab::Git::Tag.new(self, {
+            name: name,
+            target: target,
+            target_commit: target_commit
+          })
         end.compact
       end
 
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index 8a8f7b051ed..e44284572fd 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -1,17 +1,99 @@
 module Gitlab
   module Git
     class Tag < Ref
-      attr_reader :object_sha
+      extend Gitlab::EncodingHelper
+
+      attr_reader :object_sha, :repository
+
+      MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes
+      SERIALIZE_KEYS = %i[name target target_commit message].freeze
+
+      attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator
+
+      class << self
+        def get_message(repository, tag_id)
+          BatchLoader.for({ repository: repository, tag_id: tag_id }).batch do |items, loader|
+            items_by_repo = items.group_by { |i| i[:repository] }
+
+            items_by_repo.each do |repo, items|
+              tag_ids = items.map { |i| i[:tag_id] }
+
+              messages = get_messages(repository, tag_ids)
+
+              messages.each do |id, message|
+                loader.call({ repository: repository, tag_id: id }, message)
+              end
+            end
+          end
+        end
+
+        def get_messages(repository, tag_ids)
+          repository.gitaly_migrate(:tag_messages) do |is_enabled|
+            if is_enabled
+              repository.gitaly_ref_client.get_tag_messages(tag_ids)
+            else
+              tag_ids.map do |id|
+                tag = repository.rugged.lookup(id)
+                message = tag.is_a?(Rugged::Commit) ? "" : tag.message
+
+                [id, message]
+              end.to_h
+            end
+          end
+        end
+      end
+
+      def initialize(repository, raw_tag)
+        @repository = repository
+        @raw_tag = raw_tag
+
+        case raw_tag
+        when Hash
+          init_from_hash
+        when Gitaly::Tag
+          init_from_gitaly
+        end
 
-      def initialize(repository, name, target, target_commit, message = nil)
         super(repository, name, target, target_commit)
+      end
+
+      def init_from_hash
+        raw_tag = @raw_tag.symbolize_keys
+
+        SERIALIZE_KEYS.each do |key|
+          send("#{key}=", raw_tag[key]) # rubocop:disable GitlabSecurity/PublicSend
+        end
+      end
+
+      def init_from_gitaly
+        @name = encode!(@raw_tag.name.dup)
+        @target = @raw_tag.id
+        @message = message_from_gitaly_tag
 
-        @message = message
+        if @raw_tag.target_commit.present?
+          @target_commit = Gitlab::Git::Commit.decorate(repository, @raw_tag.target_commit)
+        end
       end
 
       def message
         encode! @message
       end
+
+      private
+
+      def message_from_gitaly_tag
+        return @raw_tag.message.dup if full_message_fetched_from_gitaly?
+
+        if @raw_tag.message_size > MAX_TAG_MESSAGE_DISPLAY_SIZE
+          '--tag message is too big'
+        else
+          self.class.get_message(@repository, target)
+        end
+      end
+
+      def full_message_fetched_from_gitaly?
+        @raw_tag.message.bytesize == @raw_tag.message_size
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index d75a5f15c29..1ab8c4e0229 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -131,7 +131,7 @@ module Gitlab
       def page_formatted_data(title:, dir: nil, version: nil)
         version = version&.id
 
-        @repository.gitaly_migrate(:wiki_page_formatted_data) do |is_enabled|
+        @repository.gitaly_migrate(:wiki_page_formatted_data, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version)
           else
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index a36e6c822f9..1f5f88bf792 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -334,6 +334,22 @@ module Gitlab
         signatures
       end
 
+      def get_commit_messages(commit_ids)
+        request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids)
+        response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request)
+
+        messages = Hash.new { |h, k| h[k] = ''.b }
+        current_commit_id = nil
+
+        response.each do |rpc_message|
+          current_commit_id = rpc_message.commit_id if rpc_message.commit_id.present?
+
+          messages[current_commit_id] << rpc_message.message
+        end
+
+        messages
+      end
+
       private
 
       def call_commit_diff(request_params, options = {})
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 831cfd1e014..44b0e517bf0 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -40,7 +40,7 @@ module Gitlab
           raise Gitlab::Git::Repository::TagExistsError
         end
 
-        Util.gitlab_tag_from_gitaly_tag(@repository, response.tag)
+        Gitlab::Git::Tag.new(@repository, response.tag)
       rescue GRPC::FailedPrecondition => e
         raise Gitlab::Git::Repository::InvalidRef, e
       end
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index ba6b577fd17..3ac46be6208 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -171,6 +171,22 @@ module Gitlab
         consume_ref_contains_sha_response(stream, :branch_names)
       end
 
+      def get_tag_messages(tag_ids)
+        request = Gitaly::GetTagMessagesRequest.new(repository: @gitaly_repo, tag_ids: tag_ids)
+        response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request)
+
+        messages = Hash.new { |h, k| h[k] = ''.b }
+        current_tag_id = nil
+
+        response.each do |rpc_message|
+          current_tag_id = rpc_message.tag_id if rpc_message.tag_id.present?
+
+          messages[current_tag_id] << rpc_message.message
+        end
+
+        messages
+      end
+
       private
 
       def consume_refs_response(response)
@@ -210,7 +226,7 @@ module Gitlab
 
       def consume_tags_response(response)
         response.flat_map do |message|
-          message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) }
+          message.tags.map { |gitaly_tag| Gitlab::Git::Tag.new(@repository, gitaly_tag) }
         end
       end
 
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 132a5947f17..ee01f5a5bd9 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -301,6 +301,16 @@ module Gitlab
 
         GitalyClient.call(@storage, :repository_service, :get_raw_changes, request)
       end
+
+      def search_files_by_name(ref, query)
+        request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query)
+        GitalyClient.call(@storage, :repository_service, :search_files_by_name, request).flat_map(&:files)
+      end
+
+      def search_files_by_content(ref, query)
+        request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query)
+        GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches)
+      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb
index 405567db94a..9c19c51d412 100644
--- a/lib/gitlab/gitaly_client/util.rb
+++ b/lib/gitlab/gitaly_client/util.rb
@@ -21,20 +21,6 @@ module Gitlab
                                       gitaly_repository.relative_path,
                                       gitaly_repository.gl_repository)
         end
-
-        def gitlab_tag_from_gitaly_tag(repository, gitaly_tag)
-          if gitaly_tag.target_commit.present?
-            commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit)
-          end
-
-          Gitlab::Git::Tag.new(
-            repository,
-            Gitlab::EncodingHelper.encode!(gitaly_tag.name.dup),
-            gitaly_tag.id,
-            commit,
-            Gitlab::EncodingHelper.encode!(gitaly_tag.message.chomp)
-          )
-        end
       end
     end
   end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index c741dabe168..0d31934347f 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -15,7 +15,7 @@ module Gitlab
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
       gon.sentry_dsn             = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled
       gon.gitlab_url             = Gitlab.config.gitlab.url
-      gon.revision               = Gitlab::REVISION
+      gon.revision               = Gitlab.revision
       gon.gitlab_logo            = ActionController::Base.helpers.asset_path('gitlab_logo.png')
       gon.sprite_icons           = IconsHelper.sprite_icon_path
       gon.sprite_file_icons      = IconsHelper.sprite_file_icons_path
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index e3e9f156fb4..4a41a69840b 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -28,7 +28,7 @@ module Gitlab
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
 
-      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze
+      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
 
       TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze
 
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
index c9122a23568..d323cb9dadf 100644
--- a/lib/gitlab/incoming_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -57,7 +57,7 @@ module Gitlab
 
         regex = Regexp.escape(wildcard_address)
         regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)')
-        Regexp.new(regex).freeze
+        Regexp.new(/\A#{regex}\z/).freeze
       end
     end
   end
diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb
index 89ff02a96d6..3799aaebf1c 100644
--- a/lib/gitlab/metrics/web_transaction.rb
+++ b/lib/gitlab/metrics/web_transaction.rb
@@ -4,18 +4,6 @@ module Gitlab
       CONTROLLER_KEY = 'action_controller.instance'.freeze
       ENDPOINT_KEY = 'api.endpoint'.freeze
 
-      CONTENT_TYPES = {
-        'text/html' => :html,
-        'text/plain' => :txt,
-        'application/json' => :json,
-        'text/js' => :js,
-        'application/atom+xml' => :atom,
-        'image/png' => :png,
-        'image/jpeg' => :jpeg,
-        'image/gif' => :gif,
-        'image/svg+xml' => :svg
-      }.freeze
-
       def initialize(env)
         super()
         @env = env
@@ -40,7 +28,11 @@ module Gitlab
         controller = @env[CONTROLLER_KEY]
 
         action = "#{controller.action_name}"
-        suffix = CONTENT_TYPES[controller.content_type]
+
+        # Devise exposes a method called "request_format" that does the below.
+        # However, this method is not available to all controllers (e.g. certain
+        # Doorkeeper controllers). As such we use the underlying code directly.
+        suffix = controller.request.format.try(:ref)
 
         if suffix && suffix != :html
           action += ".#{suffix}"
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 390efda326a..2e9b6e302f5 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -59,7 +59,7 @@ module Gitlab
       startline = 0
 
       result.each_line.each_with_index do |line, index|
-        prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>.*)\x00(?<startline>\d+)\x00/)&.tap do |matches|
+        prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/)&.tap do |matches|
           ref = matches[:ref]
           filename = matches[:filename]
           startline = matches[:startline]
diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb
index 9c92b83dddc..6bb00d8ae21 100644
--- a/lib/gitlab/serializer/pagination.rb
+++ b/lib/gitlab/serializer/pagination.rb
@@ -17,8 +17,6 @@ module Gitlab
         end
       end
 
-      private
-
       # Methods needed by `API::Helpers::Pagination`
       #
 
diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb
index 7ce2e9d636e..dc2d91dfa23 100644
--- a/lib/gitlab/untrusted_regexp.rb
+++ b/lib/gitlab/untrusted_regexp.rb
@@ -9,9 +9,15 @@ module Gitlab
   # there is a strict limit on total execution time. See the RE2 documentation
   # at https://github.com/google/re2/wiki/Syntax for more details.
   class UntrustedRegexp
-    delegate :===, to: :regexp
+    require_dependency 're2'
+
+    delegate :===, :source, to: :regexp
+
+    def initialize(pattern, multiline: false)
+      if multiline
+        pattern = "(?m)#{pattern}"
+      end
 
-    def initialize(pattern)
       @regexp = RE2::Regexp.new(pattern, log_errors: false)
 
       raise RegexpError.new(regexp.error) unless regexp.ok?
@@ -31,6 +37,41 @@ module Gitlab
       RE2.Replace(text, regexp, rewrite)
     end
 
+    def ==(other)
+      self.source == other.source
+    end
+
+    # Handles regular expressions with the preferred RE2 library where possible
+    # via UntustedRegex. Falls back to Ruby's built-in regular expression library
+    # when the syntax would be invalid in RE2.
+    #
+    # One difference between these is `(?m)` multi-line mode. Ruby regex enables
+    # this by default, but also handles `^` and `$` differently.
+    # See: https://www.regular-expressions.info/modifiers.html
+    def self.with_fallback(pattern, multiline: false)
+      UntrustedRegexp.new(pattern, multiline: multiline)
+    rescue RegexpError
+      Regexp.new(pattern)
+    end
+
+    def self.valid?(pattern)
+      !!self.fabricate(pattern)
+    rescue RegexpError
+      false
+    end
+
+    def self.fabricate(pattern)
+      matches = pattern.match(%r{^/(?<regexp>.+)/(?<flags>[ismU]*)$})
+
+      raise RegexpError, 'Invalid regular expression!' if matches.nil?
+
+      expression = matches[:regexp]
+      flags = matches[:flags]
+      expression.prepend("(?#{flags})") if flags.present?
+
+      self.new(expression, multiline: false)
+    end
+
     private
 
     attr_reader :regexp
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 1f060de657d..e893e46ee86 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -65,12 +65,7 @@ module Gitlab
         params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha)
         raise "Repository or ref not found" if params.empty?
 
-        if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
-          params.merge!(
-            'GitalyServer' => gitaly_server_hash(repository),
-            'GitalyRepository' => repository.gitaly_repository.to_h
-          )
-        end
+        params['GitalyServer'] = gitaly_server_hash(repository)
 
         # If present DisableCache must be a Boolean. Otherwise workhorse ignores it.
         params['DisableCache'] = true if git_archive_cache_disabled?
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 24e37f6c6cc..e96fbb64372 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -6,7 +6,6 @@ namespace :gitlab do
     desc "GitLab | Create a backup of the GitLab system"
     task create: :gitlab_environment do
       warn_user_is_not_gitlab
-      configure_cron_mode
 
       Rake::Task["gitlab:backup:db:create"].invoke
       Rake::Task["gitlab:backup:repo:create"].invoke
@@ -17,7 +16,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:lfs:create"].invoke
       Rake::Task["gitlab:backup:registry:create"].invoke
 
-      backup = Backup::Manager.new
+      backup = Backup::Manager.new(progress)
       backup.pack
       backup.cleanup
       backup.remove_old
@@ -27,9 +26,8 @@ namespace :gitlab do
     desc 'GitLab | Restore a previously created backup'
     task restore: :gitlab_environment do
       warn_user_is_not_gitlab
-      configure_cron_mode
 
-      backup = Backup::Manager.new
+      backup = Backup::Manager.new(progress)
       backup.unpack
 
       unless backup.skipped?('db')
@@ -49,9 +47,9 @@ namespace :gitlab do
 
           # Drop all tables Load the schema to ensure we don't have any newer tables
           # hanging out from a failed upgrade
-          $progress.puts 'Cleaning the database ... '.color(:blue)
+          progress.puts 'Cleaning the database ... '.color(:blue)
           Rake::Task['gitlab:db:drop_tables'].invoke
-          $progress.puts 'done'.color(:green)
+          progress.puts 'done'.color(:green)
           Rake::Task['gitlab:backup:db:restore'].invoke
         rescue Gitlab::TaskAbortedByUserError
           puts "Quitting...".color(:red)
@@ -74,173 +72,173 @@ namespace :gitlab do
 
     namespace :repo do
       task create: :gitlab_environment do
-        $progress.puts "Dumping repositories ...".color(:blue)
+        progress.puts "Dumping repositories ...".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Repository.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Repository.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring repositories ...".color(:blue)
-        Backup::Repository.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring repositories ...".color(:blue)
+        Backup::Repository.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :db do
       task create: :gitlab_environment do
-        $progress.puts "Dumping database ... ".color(:blue)
+        progress.puts "Dumping database ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("db")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Database.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Database.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring database ... ".color(:blue)
-        Backup::Database.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring database ... ".color(:blue)
+        Backup::Database.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :builds do
       task create: :gitlab_environment do
-        $progress.puts "Dumping builds ... ".color(:blue)
+        progress.puts "Dumping builds ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("builds")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Builds.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Builds.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring builds ... ".color(:blue)
-        Backup::Builds.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring builds ... ".color(:blue)
+        Backup::Builds.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :uploads do
       task create: :gitlab_environment do
-        $progress.puts "Dumping uploads ... ".color(:blue)
+        progress.puts "Dumping uploads ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Uploads.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Uploads.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring uploads ... ".color(:blue)
-        Backup::Uploads.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring uploads ... ".color(:blue)
+        Backup::Uploads.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :artifacts do
       task create: :gitlab_environment do
-        $progress.puts "Dumping artifacts ... ".color(:blue)
+        progress.puts "Dumping artifacts ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Artifacts.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Artifacts.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring artifacts ... ".color(:blue)
-        Backup::Artifacts.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring artifacts ... ".color(:blue)
+        Backup::Artifacts.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :pages do
       task create: :gitlab_environment do
-        $progress.puts "Dumping pages ... ".color(:blue)
+        progress.puts "Dumping pages ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("pages")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Pages.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Pages.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring pages ... ".color(:blue)
-        Backup::Pages.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring pages ... ".color(:blue)
+        Backup::Pages.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :lfs do
       task create: :gitlab_environment do
-        $progress.puts "Dumping lfs objects ... ".color(:blue)
+        progress.puts "Dumping lfs objects ... ".color(:blue)
 
         if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
-          $progress.puts "[SKIPPED]".color(:cyan)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          Backup::Lfs.new.dump
-          $progress.puts "done".color(:green)
+          Backup::Lfs.new(progress).dump
+          progress.puts "done".color(:green)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring lfs objects ... ".color(:blue)
-        Backup::Lfs.new.restore
-        $progress.puts "done".color(:green)
+        progress.puts "Restoring lfs objects ... ".color(:blue)
+        Backup::Lfs.new(progress).restore
+        progress.puts "done".color(:green)
       end
     end
 
     namespace :registry do
       task create: :gitlab_environment do
-        $progress.puts "Dumping container registry images ... ".color(:blue)
+        progress.puts "Dumping container registry images ... ".color(:blue)
 
         if Gitlab.config.registry.enabled
           if ENV["SKIP"] && ENV["SKIP"].include?("registry")
-            $progress.puts "[SKIPPED]".color(:cyan)
+            progress.puts "[SKIPPED]".color(:cyan)
           else
-            Backup::Registry.new.dump
-            $progress.puts "done".color(:green)
+            Backup::Registry.new(progress).dump
+            progress.puts "done".color(:green)
           end
         else
-          $progress.puts "[DISABLED]".color(:cyan)
+          progress.puts "[DISABLED]".color(:cyan)
         end
       end
 
       task restore: :gitlab_environment do
-        $progress.puts "Restoring container registry images ... ".color(:blue)
+        progress.puts "Restoring container registry images ... ".color(:blue)
 
         if Gitlab.config.registry.enabled
-          Backup::Registry.new.restore
-          $progress.puts "done".color(:green)
+          Backup::Registry.new(progress).restore
+          progress.puts "done".color(:green)
         else
-          $progress.puts "[DISABLED]".color(:cyan)
+          progress.puts "[DISABLED]".color(:cyan)
         end
       end
     end
 
-    def configure_cron_mode
+    def progress
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
         # StringIO.
         require 'stringio'
-        $progress = StringIO.new
+        StringIO.new
       else
-        $progress = $stdout
+        $stdout
       end
     end
   end # namespace end: backup
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 47ed522aec3..289aa5d9060 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -47,7 +47,7 @@ namespace :gitlab do
       puts ""
       puts "GitLab information".color(:yellow)
       puts "Version:\t#{Gitlab::VERSION}"
-      puts "Revision:\t#{Gitlab::REVISION}"
+      puts "Revision:\t#{Gitlab.revision}"
       puts "Directory:\t#{Rails.root}"
       puts "DB Adapter:\t#{database_adapter}"
       puts "URL:\t\t#{Gitlab.config.gitlab.url}"
diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake
index 523b0fa055b..2222807fe13 100644
--- a/lib/tasks/gitlab/test.rake
+++ b/lib/tasks/gitlab/test.rake
@@ -4,7 +4,6 @@ namespace :gitlab do
     cmds = [
       %w(rake brakeman),
       %w(rake rubocop),
-      %w(rake spinach),
       %w(rake spec),
       %w(rake karma)
     ]
diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake
index c6204f89de4..9b05876034c 100644
--- a/lib/tasks/migrate/add_limits_mysql.rake
+++ b/lib/tasks/migrate/add_limits_mysql.rake
@@ -2,6 +2,7 @@ require Rails.root.join('db/migrate/limits_to_mysql')
 require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
 require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
 require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
+require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
@@ -10,4 +11,5 @@ task add_limits_mysql: :environment do
   MarkdownCacheLimitsToMysql.new.up
   MergeRequestDiffFileLimitsToMysql.new.up
   LimitsCiBuildTraceChunksRawDataForMysql.new.up
+  IncreaseMysqlTextLimitForGpgKeys.new.up
 end
diff --git a/lib/tasks/migrate/composite_primary_keys.rake b/lib/tasks/migrate/composite_primary_keys.rake
new file mode 100644
index 00000000000..eb112434dd9
--- /dev/null
+++ b/lib/tasks/migrate/composite_primary_keys.rake
@@ -0,0 +1,15 @@
+namespace :gitlab do
+  namespace :db do
+    desc 'GitLab | Adds primary keys to tables that only have composite unique keys'
+    task composite_primary_keys_add: :environment do
+      require Rails.root.join('db/optional_migrations/composite_primary_keys')
+      CompositePrimaryKeysMigration.new.up
+    end
+
+    desc 'GitLab | Removes previously added composite primary keys'
+    task composite_primary_keys_drop: :environment do
+      require Rails.root.join('db/optional_migrations/composite_primary_keys')
+      CompositePrimaryKeysMigration.new.down
+    end
+  end
+end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index af30ecb0e9b..e7aab50e42a 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -8,6 +8,7 @@ task setup_postgresql: :environment do
   require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
   require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb')
   require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb')
+  require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb')
   require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb')
 
   NamespacesProjectsPathLowerIndexes.new.up
@@ -18,5 +19,6 @@ task setup_postgresql: :environment do
   IndexRedirectRoutesPathForLike.new.up
   AddIndexOnNamespacesLowerName.new.up
   UsersNameLowerIndex.new.up
+  ProjectNameLowerIndex.new.up
   AddPathIndexToRedirectRoutes.new.up
 end
diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
deleted file mode 100644
index 19ff13f06c0..00000000000
--- a/lib/tasks/spinach.rake
+++ /dev/null
@@ -1,60 +0,0 @@
-Rake::Task["spinach"].clear if Rake::Task.task_defined?('spinach')
-
-namespace :spinach do
-  namespace :project do
-    desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features"
-    task :half do
-      run_spinach_tests('@project_commits,@project_issues,@project_merge_requests')
-    end
-
-    desc "GitLab | Spinach | Run remaining project spinach features"
-    task :rest do
-      run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests')
-    end
-  end
-
-  desc "GitLab | Spinach | Run project spinach features"
-  task :project do
-    run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets')
-  end
-
-  desc "GitLab | Spinach | Run other spinach features"
-  task :other do
-    run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets')
-  end
-
-  desc "GitLab | Spinach | Run other spinach features"
-  task :builds do
-    run_spinach_tests('@builds')
-  end
-end
-
-desc "GitLab | Run spinach"
-task :spinach do
-  run_spinach_tests(nil)
-end
-
-def run_system_command(cmd)
-  system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd)
-end
-
-def run_spinach_command(args)
-  run_system_command(%w(spinach -r rerun) + args)
-end
-
-def run_spinach_tests(tags)
-  success = run_spinach_command(%W(--tags #{tags}))
-  3.times do |_|
-    break if success
-    break unless File.exist?('tmp/spinach-rerun.txt')
-
-    tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp)
-    puts ''
-    puts "Spinach tests for #{tags}: Retrying tests... #{tests}".color(:red)
-    puts ''
-    sleep(3)
-    success = run_spinach_command(tests)
-  end
-
-  raise("spinach tests for #{tags} failed!") unless success
-end