Merge branch 'zj-pool-repository-creation' into 'master'

Allow public forks to be deduplicated

See merge request gitlab-org/gitlab-ce!23508

6 files changed, 101 insertions, 0 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 672c77539af..dfce00a10a1 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -85,6 +85,10 @@
 - todos_destroyer:todos_destroyer_project_private
 - todos_destroyer:todos_destroyer_private_features
 
+- object_pool:object_pool_create
+- object_pool:object_pool_schedule_join
+- object_pool:object_pool_join
+
 - default
 - mailers # ActionMailer::DeliveryJob.queue_name
 
diff --git a/app/workers/concerns/object_pool_queue.rb b/app/workers/concerns/object_pool_queue.rb
new file mode 100644
index 00000000000..5b648df9c72
--- /dev/null
+++ b/app/workers/concerns/object_pool_queue.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+##
+# Concern for setting Sidekiq settings for the various ObjectPool queues
+#
+module ObjectPoolQueue
+  extend ActiveSupport::Concern
+
+  included do
+    queue_namespace :object_pool
+  end
+end
diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index 2d381c6fd6c..d3628b23189 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -28,6 +28,8 @@ class GitGarbageCollectWorker
     # Refresh the branch cache in case garbage collection caused a ref lookup to fail
     flush_ref_caches(project) if task == :gc
 
+    project.repository.expire_statistics_caches
+
     # In case pack files are deleted, release libgit2 cache and open file
     # descriptors ASAP instead of waiting for Ruby garbage collection
     project.cleanup
diff --git a/app/workers/object_pool/create_worker.rb b/app/workers/object_pool/create_worker.rb
new file mode 100644
index 00000000000..135b99886dc
--- /dev/null
+++ b/app/workers/object_pool/create_worker.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module ObjectPool
+  class CreateWorker
+    include ApplicationWorker
+    include ObjectPoolQueue
+    include ExclusiveLeaseGuard
+
+    attr_reader :pool
+
+    def perform(pool_id)
+      @pool = PoolRepository.find_by_id(pool_id)
+      return unless pool
+
+      try_obtain_lease do
+        perform_pool_creation
+      end
+    end
+
+    private
+
+    def perform_pool_creation
+      return unless pool.failed? || pool.scheduled?
+
+      # If this is a retry and the previous execution failed, deletion will
+      # bring the pool back to a pristine state
+      pool.delete_object_pool if pool.failed?
+
+      pool.create_object_pool
+      pool.mark_ready
+    rescue => e
+      pool.mark_failed
+      raise e
+    end
+
+    def lease_key
+      "object_pool:create:#{pool.id}"
+    end
+
+    def lease_timeout
+      1.hour
+    end
+  end
+end
diff --git a/app/workers/object_pool/join_worker.rb b/app/workers/object_pool/join_worker.rb
new file mode 100644
index 00000000000..07676011b2a
--- /dev/null
+++ b/app/workers/object_pool/join_worker.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ObjectPool
+  class JoinWorker
+    include ApplicationWorker
+    include ObjectPoolQueue
+
+    def perform(pool_id, project_id)
+      pool = PoolRepository.find_by_id(pool_id)
+      return unless pool&.joinable?
+
+      project = Project.find_by_id(project_id)
+      return unless project
+
+      pool.link_repository(project.repository)
+
+      Projects::HousekeepingService.new(project).execute
+    end
+  end
+end
diff --git a/app/workers/object_pool/schedule_join_worker.rb b/app/workers/object_pool/schedule_join_worker.rb
new file mode 100644
index 00000000000..647a8b72435
--- /dev/null
+++ b/app/workers/object_pool/schedule_join_worker.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ObjectPool
+  class ScheduleJoinWorker
+    include ApplicationWorker
+    include ObjectPoolQueue
+
+    def perform(pool_id)
+      pool = PoolRepository.find_by_id(pool_id)
+      return unless pool&.joinable?
+
+      pool.member_projects.find_each do |project|
+        next if project.forked? && !project.import_finished?
+
+        ObjectPool::JoinWorker.perform_async(pool.id, project.id)
+      end
+    end
+  end
+end