Load all projects, namespaces, routes in 1 query

Avoid doing sequential database queries to load the namespaces and the
routes of projects and namespaces.

This results in the following query:

```sql
SELECT "projects"."id" AS t0_r0,
       "projects"."name" AS t0_r1,
       "projects"."path" AS t0_r2,
       "projects"."description" AS t0_r3,
       "projects"."created_at" AS t0_r4,
       "projects"."updated_at" AS t0_r5,
       "projects"."creator_id" AS t0_r6,
       "projects"."namespace_id" AS t0_r7,
       "projects"."last_activity_at" AS t0_r8,
       "projects"."import_url" AS t0_r9,
       "projects"."visibility_level" AS t0_r10,
       "projects"."archived" AS t0_r11,
       "projects"."avatar" AS t0_r12,
       "projects"."import_status" AS t0_r13,
       "projects"."star_count" AS t0_r14,
       "projects"."import_type" AS t0_r15,
       "projects"."import_source" AS t0_r16,
       "projects"."import_error" AS t0_r17,
       "projects"."ci_id" AS t0_r18,
       "projects"."shared_runners_enabled" AS t0_r19,
       "projects"."runners_token" AS t0_r20,
       "projects"."build_coverage_regex" AS t0_r21,
       "projects"."build_allow_git_fetch" AS t0_r22,
       "projects"."build_timeout" AS t0_r23,
       "projects"."pending_delete" AS t0_r24,
       "projects"."public_builds" AS t0_r25,
       "projects"."last_repository_check_failed" AS t0_r26,
       "projects"."last_repository_check_at" AS t0_r27,
       "projects"."container_registry_enabled" AS t0_r28,
       "projects"."only_allow_merge_if_pipeline_succeeds" AS t0_r29,
       "projects"."has_external_issue_tracker" AS t0_r30,
       "projects"."repository_storage" AS t0_r31,
       "projects"."request_access_enabled" AS t0_r32,
       "projects"."has_external_wiki" AS t0_r33,
       "projects"."ci_config_path" AS t0_r34,
       "projects"."lfs_enabled" AS t0_r35,
       "projects"."description_html" AS t0_r36,
       "projects"."only_allow_merge_if_all_discussions_are_resolved" AS t0_r37,
       "projects"."printing_merge_request_link_enabled" AS t0_r38,
       "projects"."auto_cancel_pending_pipelines" AS t0_r39,
       "projects"."import_jid" AS t0_r40,
       "projects"."cached_markdown_version" AS t0_r41,
       "projects"."delete_error" AS t0_r42,
       "projects"."last_repository_updated_at" AS t0_r43,
       "projects"."storage_version" AS t0_r44,
       "projects"."resolve_outdated_diff_discussions" AS t0_r45,
       "projects"."repository_read_only" AS t0_r46,
       "projects"."merge_requests_ff_only_enabled" AS t0_r47,
       "projects"."merge_requests_rebase_enabled" AS t0_r48,
       "projects"."jobs_cache_index" AS t0_r49,
       "projects"."pages_https_only" AS t0_r50,
       "projects"."remote_mirror_available_overridden" AS t0_r51,
       "projects"."pool_repository_id" AS t0_r52,
       "projects"."runners_token_encrypted" AS t0_r53,
       "projects"."bfg_object_map" AS t0_r54,
       "namespaces"."id" AS t1_r0,
       "namespaces"."name" AS t1_r1,
       "namespaces"."path" AS t1_r2,
       "namespaces"."owner_id" AS t1_r3,
       "namespaces"."created_at" AS t1_r4,
       "namespaces"."updated_at" AS t1_r5,
       "namespaces"."type" AS t1_r6,
       "namespaces"."description" AS t1_r7,
       "namespaces"."avatar" AS t1_r8,
       "namespaces"."share_with_group_lock" AS t1_r9,
       "namespaces"."visibility_level" AS t1_r10,
       "namespaces"."request_access_enabled" AS t1_r11,
       "namespaces"."description_html" AS t1_r12,
       "namespaces"."lfs_enabled" AS t1_r13,
       "namespaces"."parent_id" AS t1_r14,
       "namespaces"."require_two_factor_authentication" AS t1_r15,
       "namespaces"."two_factor_grace_period" AS t1_r16,
       "namespaces"."cached_markdown_version" AS t1_r17,
       "namespaces"."runners_token" AS t1_r18,
       "namespaces"."runners_token_encrypted" AS t1_r19,
       "routes"."id" AS t2_r0,
       "routes"."source_id" AS t2_r1,
       "routes"."source_type" AS t2_r2,
       "routes"."path" AS t2_r3,
       "routes"."created_at" AS t2_r4,
       "routes"."updated_at" AS t2_r5,
       "routes"."name" AS t2_r6,
       "routes_projects"."id" AS t3_r0,
       "routes_projects"."source_id" AS t3_r1,
       "routes_projects"."source_type" AS t3_r2,
       "routes_projects"."path" AS t3_r3,
       "routes_projects"."created_at" AS t3_r4,
       "routes_projects"."updated_at" AS t3_r5,
       "routes_projects"."name" AS t3_r6
FROM "projects"
LEFT OUTER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id"
LEFT OUTER JOIN "routes" ON "routes"."source_id" = "namespaces"."id"
AND "routes"."source_type" = $1
LEFT OUTER JOIN "routes" "routes_projects" ON "routes_projects"."source_id" = "projects"."id"
AND "routes_projects"."source_type" = $2
LEFT OUTER JOIN "project_repositories" ON "projects"."id" = "project_repositories"."project_id"
WHERE ("projects"."storage_version" IS NULL
       OR "projects"."storage_version" = 0)
  AND "project_repositories"."project_id" IS NULL
  AND ("projects"."id" BETWEEN $3 AND $4);
  -- [["source_type", "Namespace"],
  --  ["source_type", "Project"],
  --  ["id", 1],
  --  ["id", 4]]
```

2 files changed, 31 insertions, 10 deletions

diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
index 2a7f10c21e1..d03ad813d9e 100644
--- a/lib/gitlab/background_migration/backfill_project_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -99,10 +99,12 @@ module Gitlab
         end
       end
 
+      # Route model
       class Route < ActiveRecord::Base
+        belongs_to :source, inverse_of: :route
       end
 
-      # Namespace model.
+      # Namespace model
       class Namespace < ActiveRecord::Base
         self.table_name = 'namespaces'
         self.inheritance_column = nil
@@ -111,12 +113,10 @@ module Gitlab
 
         belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces'
 
+        has_one :route, -> { where(source_type: 'Namespace') }, inverse_of: :source, foreign_key: :source_id
+
         has_many :projects, inverse_of: :parent
         has_many :namespaces, inverse_of: :parent
-
-        def route
-          Route.find_by(source_type: 'Namespace', source_id: self.id)
-        end
       end
 
       # ProjectRegistry model
@@ -141,6 +141,7 @@ module Gitlab
 
         belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects'
 
+        has_one :route, -> { where(source_type: 'Project') }, inverse_of: :source, foreign_key: :source_id
         has_one :project_repository, inverse_of: :project
 
         delegate :disk_path, to: :storage
@@ -172,10 +173,6 @@ module Gitlab
           end
         end
 
-        def route
-          Route.find_by(source_type: 'Project', source_id: self.id)
-        end
-
         def storage
           @storage ||=
             if hashed_storage?
@@ -192,7 +189,11 @@ module Gitlab
       end
 
       def perform(start_id, stop_id)
+        ActiveRecord::Base.logger = Logger.new(STDOUT)
+
         Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id))
+
+        ActiveRecord::Base.logger = nil
       end
 
       private
@@ -203,13 +204,31 @@ module Gitlab
       end
 
       def project_repositories(start_id, stop_id)
+#          .eager_load(:route, :parent, parent: [:route])
+#          .includes(:parent, :route, parent: [:route]).references(:namespaces)
+#          .includes(:parent).references(:namespaces)
+#          .joins(*routes_joins).references(:routes)
+
+
         projects
           .without_project_repository
+          .includes(:route, parent: [:route]).references(:routes)
+          .includes(:parent).references(:namespaces)
           .where(id: start_id..stop_id)
           .map { |project| build_attributes_for_project(project) }
           .compact
       end
 
+      def routes_joins
+        routes = Route.arel_table
+        projects = Project.arel_table
+        routes_projects = routes.alias('routes_projects')
+
+        projects.join(routes, Arel::Nodes::OuterJoin).on(projects[:namespace_id].eq(routes[:source_id]).and(routes[:source_type].eq('Namespace')))
+          .join(routes_projects, Arel::Nodes::OuterJoin).on(projects[:id].eq(routes_projects[:source_id]).and(routes_projects[:source_type].eq('Project')))
+          .join_sources
+      end
+
       def build_attributes_for_project(project)
         {
           project_id: project.id,
diff --git a/spec/support/shared_examples/lib/gitlab/background_migration/backfill_project_repositories_examples.rb b/spec/support/shared_examples/lib/gitlab/background_migration/backfill_project_repositories_examples.rb
index 1f688c0f9d3..dcf7c1a90c2 100644
--- a/spec/support/shared_examples/lib/gitlab/background_migration/backfill_project_repositories_examples.rb
+++ b/spec/support/shared_examples/lib/gitlab/background_migration/backfill_project_repositories_examples.rb
@@ -32,11 +32,13 @@ shared_examples 'backfill migration for project repositories' do |storage|
 
     it 'inserts rows in a single query' do
       projects.create!(name: 'foo', path: 'foo', namespace_id: group.id, storage_version: storage_version, repository_storage: shard.name)
+      group2 = namespaces.create!(name: 'gro', path: 'gro')
 
       control_count = ActiveRecord::QueryRecorder.new { described_class.new.perform(1, projects.last.id) }
 
       projects.create!(name: 'bar', path: 'bar', namespace_id: group.id, storage_version: storage_version, repository_storage: shard.name)
-      projects.create!(name: 'zoo', path: 'zoo', namespace_id: group.id, storage_version: storage_version, repository_storage: shard.name)
+      projects.create!(name: 'top', path: 'top', namespace_id: group.id, storage_version: storage_version, repository_storage: shard.name)
+      projects.create!(name: 'zoo', path: 'zoo', namespace_id: group2.id, storage_version: storage_version, repository_storage: shard.name)
 
       expect { described_class.new.perform(1, projects.last.id) }.not_to exceed_query_limit(control_count)
     end