diff options
author | Toon Claes <toon@gitlab.com> | 2019-01-09 10:47:24 +0100 |
---|---|---|
committer | Toon Claes <toon@gitlab.com> | 2019-01-11 15:36:57 +0100 |
commit | d3e028b85f3f380239394f9ebf274357abf7142a (patch) | |
tree | 5c8377cacf0daf7513fecd8a59d8d6cd9fb5d931 /lib | |
parent | 618875c58c50b0c9dcfa90033ebe975004f401a2 (diff) | |
download | gitlab-ce-d3e028b85f3f380239394f9ebf274357abf7142a.tar.gz |
Load all projects, namespaces, routes in 1 query
Avoid doing sequential database queries to load the namespaces and the
routes of projects and namespaces.
This results in the following query:
```sql
SELECT "projects"."id" AS t0_r0,
"projects"."name" AS t0_r1,
"projects"."path" AS t0_r2,
"projects"."description" AS t0_r3,
"projects"."created_at" AS t0_r4,
"projects"."updated_at" AS t0_r5,
"projects"."creator_id" AS t0_r6,
"projects"."namespace_id" AS t0_r7,
"projects"."last_activity_at" AS t0_r8,
"projects"."import_url" AS t0_r9,
"projects"."visibility_level" AS t0_r10,
"projects"."archived" AS t0_r11,
"projects"."avatar" AS t0_r12,
"projects"."import_status" AS t0_r13,
"projects"."star_count" AS t0_r14,
"projects"."import_type" AS t0_r15,
"projects"."import_source" AS t0_r16,
"projects"."import_error" AS t0_r17,
"projects"."ci_id" AS t0_r18,
"projects"."shared_runners_enabled" AS t0_r19,
"projects"."runners_token" AS t0_r20,
"projects"."build_coverage_regex" AS t0_r21,
"projects"."build_allow_git_fetch" AS t0_r22,
"projects"."build_timeout" AS t0_r23,
"projects"."pending_delete" AS t0_r24,
"projects"."public_builds" AS t0_r25,
"projects"."last_repository_check_failed" AS t0_r26,
"projects"."last_repository_check_at" AS t0_r27,
"projects"."container_registry_enabled" AS t0_r28,
"projects"."only_allow_merge_if_pipeline_succeeds" AS t0_r29,
"projects"."has_external_issue_tracker" AS t0_r30,
"projects"."repository_storage" AS t0_r31,
"projects"."request_access_enabled" AS t0_r32,
"projects"."has_external_wiki" AS t0_r33,
"projects"."ci_config_path" AS t0_r34,
"projects"."lfs_enabled" AS t0_r35,
"projects"."description_html" AS t0_r36,
"projects"."only_allow_merge_if_all_discussions_are_resolved" AS t0_r37,
"projects"."printing_merge_request_link_enabled" AS t0_r38,
"projects"."auto_cancel_pending_pipelines" AS t0_r39,
"projects"."import_jid" AS t0_r40,
"projects"."cached_markdown_version" AS t0_r41,
"projects"."delete_error" AS t0_r42,
"projects"."last_repository_updated_at" AS t0_r43,
"projects"."storage_version" AS t0_r44,
"projects"."resolve_outdated_diff_discussions" AS t0_r45,
"projects"."repository_read_only" AS t0_r46,
"projects"."merge_requests_ff_only_enabled" AS t0_r47,
"projects"."merge_requests_rebase_enabled" AS t0_r48,
"projects"."jobs_cache_index" AS t0_r49,
"projects"."pages_https_only" AS t0_r50,
"projects"."remote_mirror_available_overridden" AS t0_r51,
"projects"."pool_repository_id" AS t0_r52,
"projects"."runners_token_encrypted" AS t0_r53,
"projects"."bfg_object_map" AS t0_r54,
"namespaces"."id" AS t1_r0,
"namespaces"."name" AS t1_r1,
"namespaces"."path" AS t1_r2,
"namespaces"."owner_id" AS t1_r3,
"namespaces"."created_at" AS t1_r4,
"namespaces"."updated_at" AS t1_r5,
"namespaces"."type" AS t1_r6,
"namespaces"."description" AS t1_r7,
"namespaces"."avatar" AS t1_r8,
"namespaces"."share_with_group_lock" AS t1_r9,
"namespaces"."visibility_level" AS t1_r10,
"namespaces"."request_access_enabled" AS t1_r11,
"namespaces"."description_html" AS t1_r12,
"namespaces"."lfs_enabled" AS t1_r13,
"namespaces"."parent_id" AS t1_r14,
"namespaces"."require_two_factor_authentication" AS t1_r15,
"namespaces"."two_factor_grace_period" AS t1_r16,
"namespaces"."cached_markdown_version" AS t1_r17,
"namespaces"."runners_token" AS t1_r18,
"namespaces"."runners_token_encrypted" AS t1_r19,
"routes"."id" AS t2_r0,
"routes"."source_id" AS t2_r1,
"routes"."source_type" AS t2_r2,
"routes"."path" AS t2_r3,
"routes"."created_at" AS t2_r4,
"routes"."updated_at" AS t2_r5,
"routes"."name" AS t2_r6,
"routes_projects"."id" AS t3_r0,
"routes_projects"."source_id" AS t3_r1,
"routes_projects"."source_type" AS t3_r2,
"routes_projects"."path" AS t3_r3,
"routes_projects"."created_at" AS t3_r4,
"routes_projects"."updated_at" AS t3_r5,
"routes_projects"."name" AS t3_r6
FROM "projects"
LEFT OUTER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id"
LEFT OUTER JOIN "routes" ON "routes"."source_id" = "namespaces"."id"
AND "routes"."source_type" = $1
LEFT OUTER JOIN "routes" "routes_projects" ON "routes_projects"."source_id" = "projects"."id"
AND "routes_projects"."source_type" = $2
LEFT OUTER JOIN "project_repositories" ON "projects"."id" = "project_repositories"."project_id"
WHERE ("projects"."storage_version" IS NULL
OR "projects"."storage_version" = 0)
AND "project_repositories"."project_id" IS NULL
AND ("projects"."id" BETWEEN $3 AND $4);
-- [["source_type", "Namespace"],
-- ["source_type", "Project"],
-- ["id", 1],
-- ["id", 4]]
```
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/background_migration/backfill_project_repositories.rb | 37 |
1 files changed, 28 insertions, 9 deletions
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index 2a7f10c21e1..d03ad813d9e 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -99,10 +99,12 @@ module Gitlab end end + # Route model class Route < ActiveRecord::Base + belongs_to :source, inverse_of: :route end - # Namespace model. + # Namespace model class Namespace < ActiveRecord::Base self.table_name = 'namespaces' self.inheritance_column = nil @@ -111,12 +113,10 @@ module Gitlab belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' + has_one :route, -> { where(source_type: 'Namespace') }, inverse_of: :source, foreign_key: :source_id + has_many :projects, inverse_of: :parent has_many :namespaces, inverse_of: :parent - - def route - Route.find_by(source_type: 'Namespace', source_id: self.id) - end end # ProjectRegistry model @@ -141,6 +141,7 @@ module Gitlab belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' + has_one :route, -> { where(source_type: 'Project') }, inverse_of: :source, foreign_key: :source_id has_one :project_repository, inverse_of: :project delegate :disk_path, to: :storage @@ -172,10 +173,6 @@ module Gitlab end end - def route - Route.find_by(source_type: 'Project', source_id: self.id) - end - def storage @storage ||= if hashed_storage? @@ -192,7 +189,11 @@ module Gitlab end def perform(start_id, stop_id) + ActiveRecord::Base.logger = Logger.new(STDOUT) + Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) + + ActiveRecord::Base.logger = nil end private @@ -203,13 +204,31 @@ module Gitlab end def project_repositories(start_id, stop_id) +# .eager_load(:route, :parent, parent: [:route]) +# .includes(:parent, :route, parent: [:route]).references(:namespaces) +# .includes(:parent).references(:namespaces) +# .joins(*routes_joins).references(:routes) + + projects .without_project_repository + .includes(:route, parent: [:route]).references(:routes) + .includes(:parent).references(:namespaces) .where(id: start_id..stop_id) .map { |project| build_attributes_for_project(project) } .compact end + def routes_joins + routes = Route.arel_table + projects = Project.arel_table + routes_projects = routes.alias('routes_projects') + + projects.join(routes, Arel::Nodes::OuterJoin).on(projects[:namespace_id].eq(routes[:source_id]).and(routes[:source_type].eq('Namespace'))) + .join(routes_projects, Arel::Nodes::OuterJoin).on(projects[:id].eq(routes_projects[:source_id]).and(routes_projects[:source_type].eq('Project'))) + .join_sources + end + def build_attributes_for_project(project) { project_id: project.id, |