From 211a8c3361ccf4eb92f36edbdcf15c98fcdcc8b7 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 7 Feb 2020 12:09:13 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- app/services/projects/create_service.rb | 8 +- app/services/projects/fork_service.rb | 12 +-- .../projects/lfs_pointers/lfs_download_service.rb | 6 +- .../projects/move_lfs_objects_projects_service.rb | 2 +- .../projects/propagate_instance_level_service.rb | 107 +++++++++++++++++++++ .../projects/propagate_service_template.rb | 107 --------------------- app/services/projects/unlink_fork_service.rb | 4 + 7 files changed, 120 insertions(+), 126 deletions(-) create mode 100644 app/services/projects/propagate_instance_level_service.rb delete mode 100644 app/services/projects/propagate_service_template.rb (limited to 'app/services/projects') diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index ef06545b27d..f3666f100a3 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -134,7 +134,7 @@ module Projects if @project.save unless @project.gitlab_project_import? - create_services_from_active_templates(@project) + create_services_from_active_instance_level_services(@project) @project.create_labels end @@ -160,9 +160,9 @@ module Projects end # rubocop: disable CodeReuse/ActiveRecord - def create_services_from_active_templates(project) - Service.where(template: true, active: true).each do |template| - service = Service.build_from_template(project.id, template) + def create_services_from_active_instance_level_services(project) + Service.where(instance: true, active: true).each do |template| + service = Service.build_from_instance(project.id, template) service.save! end end diff --git a/app/services/projects/fork_service.rb b/app/services/projects/fork_service.rb index e66a0ed181a..fcfea567885 100644 --- a/app/services/projects/fork_service.rb +++ b/app/services/projects/fork_service.rb @@ -26,17 +26,7 @@ module Projects build_fork_network_member(fork_to_project) - if link_fork_network(fork_to_project) - # A forked project stores its LFS objects in the `forked_from_project`. - # So the LFS objects become inaccessible, and therefore delete them from - # the database so they'll get cleaned up. - # - # TODO: refactor this to get the correct lfs objects when implementing - # https://gitlab.com/gitlab-org/gitlab-foss/issues/39769 - fork_to_project.lfs_objects_projects.delete_all - - fork_to_project - end + fork_to_project if link_fork_network(fork_to_project) end def fork_new_project diff --git a/app/services/projects/lfs_pointers/lfs_download_service.rb b/app/services/projects/lfs_pointers/lfs_download_service.rb index a009f479d5d..bd70012c76c 100644 --- a/app/services/projects/lfs_pointers/lfs_download_service.rb +++ b/app/services/projects/lfs_pointers/lfs_download_service.rb @@ -39,9 +39,9 @@ module Projects def download_lfs_file! with_tmp_file do |tmp_file| download_and_save_file!(tmp_file) - project.all_lfs_objects << LfsObject.new(oid: lfs_oid, - size: lfs_size, - file: tmp_file) + project.lfs_objects << LfsObject.new(oid: lfs_oid, + size: lfs_size, + file: tmp_file) success end diff --git a/app/services/projects/move_lfs_objects_projects_service.rb b/app/services/projects/move_lfs_objects_projects_service.rb index 10e19014db4..8cc420d7ba7 100644 --- a/app/services/projects/move_lfs_objects_projects_service.rb +++ b/app/services/projects/move_lfs_objects_projects_service.rb @@ -16,7 +16,7 @@ module Projects private def move_lfs_objects_projects - non_existent_lfs_objects_projects.update_all(project_id: @project.lfs_storage_project.id) + non_existent_lfs_objects_projects.update_all(project_id: @project.id) end def remove_remaining_lfs_objects_project diff --git a/app/services/projects/propagate_instance_level_service.rb b/app/services/projects/propagate_instance_level_service.rb new file mode 100644 index 00000000000..dc75977ba0f --- /dev/null +++ b/app/services/projects/propagate_instance_level_service.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module Projects + class PropagateInstanceLevelService + BATCH_SIZE = 100 + + def self.propagate(*args) + new(*args).propagate + end + + def initialize(instance_level_service) + @instance_level_service = instance_level_service + end + + def propagate + return unless @instance_level_service.active? + + Rails.logger.info("Propagating services for instance_level_service #{@instance_level_service.id}") # rubocop:disable Gitlab/RailsLogger + + propagate_projects_with_instance_level_service + end + + private + + def propagate_projects_with_instance_level_service + loop do + batch = Project.uncached { project_ids_batch } + + bulk_create_from_instance_level_service(batch) unless batch.empty? + + break if batch.size < BATCH_SIZE + end + end + + def bulk_create_from_instance_level_service(batch) + service_list = batch.map do |project_id| + service_hash.values << project_id + end + + Project.transaction do + bulk_insert_services(service_hash.keys << 'project_id', service_list) + run_callbacks(batch) + end + end + + def project_ids_batch + Project.connection.select_values( + <<-SQL + SELECT id + FROM projects + WHERE NOT EXISTS ( + SELECT true + FROM services + WHERE services.project_id = projects.id + AND services.type = '#{@instance_level_service.type}' + ) + AND projects.pending_delete = false + AND projects.archived = false + LIMIT #{BATCH_SIZE} + SQL + ) + end + + def bulk_insert_services(columns, values_array) + ActiveRecord::Base.connection.execute( + <<-SQL.strip_heredoc + INSERT INTO services (#{columns.join(', ')}) + VALUES #{values_array.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + SQL + ) + end + + def service_hash + @service_hash ||= + begin + instance_hash = @instance_level_service.as_json(methods: :type).except('id', 'instance', 'project_id') + + instance_hash.each_with_object({}) do |(key, value), service_hash| + value = value.is_a?(Hash) ? value.to_json : value + + service_hash[ActiveRecord::Base.connection.quote_column_name(key)] = + ActiveRecord::Base.connection.quote(value) + end + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def run_callbacks(batch) + if active_external_issue_tracker? + Project.where(id: batch).update_all(has_external_issue_tracker: true) + end + + if active_external_wiki? + Project.where(id: batch).update_all(has_external_wiki: true) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def active_external_issue_tracker? + @instance_level_service.issue_tracker? && !@instance_level_service.default + end + + def active_external_wiki? + @instance_level_service.type == 'ExternalWikiService' + end + end +end diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb deleted file mode 100644 index 6013b00b8c6..00000000000 --- a/app/services/projects/propagate_service_template.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -module Projects - class PropagateServiceTemplate - BATCH_SIZE = 100 - - def self.propagate(*args) - new(*args).propagate - end - - def initialize(template) - @template = template - end - - def propagate - return unless @template.active? - - Rails.logger.info("Propagating services for template #{@template.id}") # rubocop:disable Gitlab/RailsLogger - - propagate_projects_with_template - end - - private - - def propagate_projects_with_template - loop do - batch = Project.uncached { project_ids_batch } - - bulk_create_from_template(batch) unless batch.empty? - - break if batch.size < BATCH_SIZE - end - end - - def bulk_create_from_template(batch) - service_list = batch.map do |project_id| - service_hash.values << project_id - end - - Project.transaction do - bulk_insert_services(service_hash.keys << 'project_id', service_list) - run_callbacks(batch) - end - end - - def project_ids_batch - Project.connection.select_values( - <<-SQL - SELECT id - FROM projects - WHERE NOT EXISTS ( - SELECT true - FROM services - WHERE services.project_id = projects.id - AND services.type = '#{@template.type}' - ) - AND projects.pending_delete = false - AND projects.archived = false - LIMIT #{BATCH_SIZE} - SQL - ) - end - - def bulk_insert_services(columns, values_array) - ActiveRecord::Base.connection.execute( - <<-SQL.strip_heredoc - INSERT INTO services (#{columns.join(', ')}) - VALUES #{values_array.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} - SQL - ) - end - - def service_hash - @service_hash ||= - begin - template_hash = @template.as_json(methods: :type).except('id', 'template', 'project_id') - - template_hash.each_with_object({}) do |(key, value), service_hash| - value = value.is_a?(Hash) ? value.to_json : value - - service_hash[ActiveRecord::Base.connection.quote_column_name(key)] = - ActiveRecord::Base.connection.quote(value) - end - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def run_callbacks(batch) - if active_external_issue_tracker? - Project.where(id: batch).update_all(has_external_issue_tracker: true) - end - - if active_external_wiki? - Project.where(id: batch).update_all(has_external_wiki: true) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def active_external_issue_tracker? - @template.issue_tracker? && !@template.default - end - - def active_external_wiki? - @template.type == 'ExternalWikiService' - end - end -end diff --git a/app/services/projects/unlink_fork_service.rb b/app/services/projects/unlink_fork_service.rb index e7e0141099e..b3cf27373cd 100644 --- a/app/services/projects/unlink_fork_service.rb +++ b/app/services/projects/unlink_fork_service.rb @@ -52,6 +52,10 @@ module Projects Projects::ForksCountService.new(project).refresh_cache end + # TODO: Remove this method once all LfsObjectsProject records are backfilled + # for forks. + # + # See https://gitlab.com/gitlab-org/gitlab/issues/122002 for more info. def save_lfs_objects return unless @project.forked? -- cgit v1.2.1