diff options
Diffstat (limited to 'lib/gitlab/github_import/parallel_scheduling.rb')
-rw-r--r-- | lib/gitlab/github_import/parallel_scheduling.rb | 41 |
1 files changed, 41 insertions, 0 deletions
diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index a8e006ea082..4dec9543a13 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -72,6 +72,14 @@ module Gitlab # Imports all objects in parallel by scheduling a Sidekiq job for every # individual object. def parallel_import + if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present? + spread_parallel_import + else + parallel_import_deprecated + end + end + + def parallel_import_deprecated waiter = JobWaiter.new each_object_to_import do |object| @@ -86,6 +94,33 @@ module Gitlab waiter end + def spread_parallel_import + waiter = JobWaiter.new + + import_arguments = [] + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + import_arguments << [project.id, repr.to_hash, waiter.key] + + waiter.jobs_remaining += 1 + end + + # rubocop:disable Scalability/BulkPerformWithContext + Gitlab::ApplicationContext.with_context(project: project) do + sidekiq_worker_class.bulk_perform_in( + 1.second, + import_arguments, + batch_size: parallel_import_batch[:size], + batch_delay: parallel_import_batch[:delay] + ) + end + # rubocop:enable Scalability/BulkPerformWithContext + + waiter + end + # The method that will be called for traversing through all the objects to # import, yielding them to the supplied block. def each_object_to_import @@ -171,6 +206,12 @@ module Gitlab raise NotImplementedError end + # Default batch settings for parallel import (can be redefined in Importer classes) + # Example: { size: 100, delay: 1.minute } + def parallel_import_batch + {} + end + def abort_on_failure false end |