Use limit for search count queriessearch-100

Search query is especially slow if a user searches a generic string
which matches many records, in such case search can take tens of
seconds or time out. To speed up the search query, we search only for
first 1000 records, if there is >1000 matching records we just display
"1000+" instead of precise total count supposing that with such amount
the exact count is not so important for the user.

Because for issues even limited search was not fast enough, 2-phase
approach is used for issues: first we use simpler/faster query to get
all public issues, if this exceeds the limit, we just return the limit.
If the amount of matching results is lower than limit, we re-run more
complex search query (which includes also confidential issues).
Re-running the complex query should be fast enough in such case because the
amount of matching issues is lower than limit.

Because exact total_count is now limited, this patch also switches to
to "prev/next" pagination.

Related #40540

4 files changed, 51 insertions, 21 deletions

diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index 446f804124b..a7f0813bf74 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -175,7 +175,7 @@ module API
         end
         get "/search/:query", requirements: { query: /[^\/]+/ } do
           search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
-          projects = search_service.objects('projects', params[:page])
+          projects = search_service.objects('projects', params[:page], false)
           projects = projects.reorder(params[:order_by] => params[:sort])
 
           present paginate(projects), with: ::API::V3::Entities::Project
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 7771b15069b..4823f703ba4 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -20,7 +20,7 @@ module Gitlab
       when 'commits'
         Kaminari.paginate_array(commits).page(page).per(per_page)
       else
-        super
+        super(scope, page, false)
       end
     end
 
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 70b639501fd..7362514167f 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -40,19 +40,21 @@ module Gitlab
       @default_project_filter = default_project_filter
     end
 
-    def objects(scope, page = nil)
-      case scope
-      when 'projects'
-        projects.page(page).per(per_page)
-      when 'issues'
-        issues.page(page).per(per_page)
-      when 'merge_requests'
-        merge_requests.page(page).per(per_page)
-      when 'milestones'
-        milestones.page(page).per(per_page)
-      else
-        Kaminari.paginate_array([]).page(page).per(per_page)
-      end
+    def objects(scope, page = nil, without_count = true)
+      collection = case scope
+                   when 'projects'
+                     projects.page(page).per(per_page)
+                   when 'issues'
+                     issues.page(page).per(per_page)
+                   when 'merge_requests'
+                     merge_requests.page(page).per(per_page)
+                   when 'milestones'
+                     milestones.page(page).per(per_page)
+                   else
+                     Kaminari.paginate_array([]).page(page).per(per_page)
+                   end
+
+      without_count ? collection.without_count : collection
     end
 
     def projects_count
@@ -71,18 +73,46 @@ module Gitlab
       @milestones_count ||= milestones.count
     end
 
+    def limited_projects_count
+      @limited_projects_count ||= projects.limit(count_limit).count
+    end
+
+    def limited_issues_count
+      return @limited_issues_count if @limited_issues_count
+
+      # By default getting limited count (e.g. 1000+) is fast on issuable
+      # collections except for issues, where filtering both not confidential
+      # and confidential issues user has access to, is too complex.
+      # It's faster to try to fetch all public issues first, then only
+      # if necessary try to fetch all issues.
+      sum = issues(public_only: true).limit(count_limit).count
+      @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum
+    end
+
+    def limited_merge_requests_count
+      @limited_merge_requests_count ||= merge_requests.limit(count_limit).count
+    end
+
+    def limited_milestones_count
+      @limited_milestones_count ||= milestones.limit(count_limit).count
+    end
+
     def single_commit_result?
       false
     end
 
+    def count_limit
+      1001
+    end
+
     private
 
     def projects
       limit_projects.search(query)
     end
 
-    def issues
-      issues = IssuesFinder.new(current_user).execute
+    def issues(finder_params = {})
+      issues = IssuesFinder.new(current_user, finder_params).execute
       unless default_project_filter
         issues = issues.where(project_id: project_ids_relation)
       end
@@ -94,13 +124,13 @@ module Gitlab
           issues.full_search(query)
         end
 
-      issues.order('updated_at DESC')
+      issues.reorder('updated_at DESC')
     end
 
     def milestones
       milestones = Milestone.where(project_id: project_ids_relation)
       milestones = milestones.search(query)
-      milestones.order('updated_at DESC')
+      milestones.reorder('updated_at DESC')
     end
 
     def merge_requests
@@ -116,7 +146,7 @@ module Gitlab
           merge_requests.full_search(query)
         end
 
-      merge_requests.order('updated_at DESC')
+      merge_requests.reorder('updated_at DESC')
     end
 
     def default_scope
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index b85f70e450e..4f86b3e8f73 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -16,7 +16,7 @@ module Gitlab
       when 'snippet_blobs'
         snippet_blobs.page(page).per(per_page)
       else
-        super
+        super(scope, nil, false)
       end
     end