Archive repositories in background worker.

6 files changed, 121 insertions, 16 deletions

diff --git a/Procfile b/Procfile
index a0ab4a734a4..799b92729fa 100644
--- a/Procfile
+++ b/Procfile
@@ -1,2 +1,2 @@
 web: bundle exec unicorn_rails -p ${PORT:="3000"} -E ${RAILS_ENV:="development"} -c ${UNICORN_CONFIG:="config/unicorn.rb"}
-worker: bundle exec sidekiq -q post_receive -q mailer -q system_hook -q project_web_hook -q gitlab_shell -q common -q default
+worker: bundle exec sidekiq -q post_receive -q mailer -q archive_repo -q system_hook -q project_web_hook -q gitlab_shell -q common -q default
diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb
index cbb888b25e8..112365ba91a 100644
--- a/app/controllers/projects/repositories_controller.rb
+++ b/app/controllers/projects/repositories_controller.rb
@@ -15,14 +15,18 @@ class Projects::RepositoriesController < Projects::ApplicationController
       render_404 and return
     end
 
-    file_path = ArchiveRepositoryService.new.execute(@project, params[:ref], params[:format])
+    begin
+      file_path = ArchiveRepositoryService.new(@project, params[:ref], params[:format]).execute
+    rescue
+      return render_404
+    end
 
     if file_path
       # Send file to user
       response.headers["Content-Length"] = File.open(file_path).size.to_s
       send_file file_path
     else
-      render_404
+      redirect_to request.fullpath
     end
   end
 end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 77765cae1a0..72769498872 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -267,6 +267,9 @@ class Repository
   # Remove archives older than 2 hours
   def clean_old_archives
     repository_downloads_path = Gitlab.config.gitlab.repository_downloads_path
+
+    return unless File.directory?(repository_downloads_path)
+
     Gitlab::Popen.popen(%W(find #{repository_downloads_path} -not -path #{repository_downloads_path} -mmin +120 -delete))
   end
 
diff --git a/app/services/archive_repository_service.rb b/app/services/archive_repository_service.rb
index 8823f6fdc67..cb2026fedba 100644
--- a/app/services/archive_repository_service.rb
+++ b/app/services/archive_repository_service.rb
@@ -1,14 +1,65 @@
 class ArchiveRepositoryService
-  def execute(project, ref, format)
-    storage_path = Gitlab.config.gitlab.repository_downloads_path
+  attr_reader :project, :ref, :format
 
-    unless File.directory?(storage_path)
-      FileUtils.mkdir_p(storage_path)
+  def initialize(project, ref, format)
+    format ||= 'tar.gz'
+    @project, @ref, @format = project, ref, format
+  end
+
+  def execute
+    project.repository.clean_old_archives
+
+    raise "No archive file path" unless file_path
+
+    return file_path if archived?
+
+    unless archiving?
+      RepositoryArchiveWorker.perform_async(project.id, ref, format)
     end
 
-    format ||= 'tar.gz'
-    repository = project.repository
-    repository.clean_old_archives
-    repository.archive_repo(ref, storage_path, format.downcase)
+    archived = wait_until_archived
+
+    file_path if archived
+  end
+
+  private
+
+  def storage_path
+    Gitlab.config.gitlab.repository_downloads_path
+  end
+
+  def archive_args
+    @archive_args ||= [ref, storage_path, format.downcase]
+  end
+
+  def file_path
+    @file_path ||= project.repository.archive_file_path(*archive_args)
+  end
+
+  def pid_file_path
+    @pid_file_path ||= project.repository.archive_pid_file_path(*archive_args)
+  end
+
+  def archived?
+    File.exist?(file_path)
+  end
+
+  def archiving?
+    File.exist?(pid_file_path)
+  end
+
+  def wait_until_archived
+    timeout = 5.0
+    t1 = Time.now
+
+    begin
+      sleep 0.1
+
+      success = archived?
+
+      t2 = Time.now
+    end until success || t2 - t1 >= timeout
+
+    success
   end
 end
diff --git a/app/workers/repository_archive_worker.rb b/app/workers/repository_archive_worker.rb
new file mode 100644
index 00000000000..3f4681a80f4
--- /dev/null
+++ b/app/workers/repository_archive_worker.rb
@@ -0,0 +1,46 @@
+class RepositoryArchiveWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :archive_repo
+
+  attr_accessor :project, :ref, :format
+
+  def perform(project_id, ref, format)
+    @project = Project.find(project_id)
+    @ref, @format = ref, format
+
+    repository = project.repository
+
+    repository.clean_old_archives
+
+    return if archived? || archiving?
+
+    repository.archive_repo(*archive_args)
+  end
+
+  private
+
+  def storage_path
+    Gitlab.config.gitlab.repository_downloads_path
+  end
+
+  def archive_args
+    @archive_args ||= [ref, storage_path, format.downcase]
+  end
+
+  def file_path
+    @file_path ||= project.repository.archive_file_path(*archive_args)
+  end
+
+  def pid_file_path
+    @pid_file_path ||= project.repository.archive_pid_file_path(*archive_args)
+  end
+
+  def archived?
+    File.exist?(file_path)
+  end
+
+  def archiving?
+    File.exist?(pid_file_path)
+  end
+end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index b259914a01c..1fbf3dca3c6 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -133,10 +133,11 @@ module API
         authorize! :download_code, user_project
 
         begin
-          file_path = ArchiveRepositoryService.new.execute(
-              user_project,
-              params[:sha],
-              params[:format])
+          file_path = ArchiveRepositoryService.new(
+            user_project,
+            params[:sha],
+            params[:format]
+          ).execute
         rescue
           not_found!('File')
         end
@@ -149,7 +150,7 @@ module API
           env['api.format'] = :binary
           present data
         else
-          not_found!('File')
+          redirect request.fullpath
         end
       end