Pass commit data to ProcessCommitWorkerprocess-commit-worker-improvements

By passing commit data to this worker we remove the need for querying
the Git repository for every job. This in turn reduces the time spent
processing each job.

The migration included migrates jobs from the old format to the new
format. For this to work properly it requires downtime as otherwise
workers may start producing errors until they're using a newer version
of the worker code.

1 files changed, 92 insertions, 0 deletions

diff --git a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
new file mode 100644
index 00000000000..453a44e271a
--- /dev/null
+++ b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb
@@ -0,0 +1,92 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  class Project < ActiveRecord::Base
+    def self.find_including_path(id)
+      select("projects.*, CONCAT(namespaces.path, '/', projects.path) AS path_with_namespace").
+        joins('INNER JOIN namespaces ON namespaces.id = projects.namespace_id').
+        find_by(id: id)
+    end
+
+    def repository_storage_path
+      Gitlab.config.repositories.storages[repository_storage]
+    end
+
+    def repository_path
+      File.join(repository_storage_path, read_attribute(:path_with_namespace) + '.git')
+    end
+
+    def repository
+      @repository ||= Rugged::Repository.new(repository_path)
+    end
+  end
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Existing workers will error until they are using a newer version of the code'
+
+  disable_ddl_transaction!
+
+  def up
+    Sidekiq.redis do |redis|
+      new_jobs = []
+
+      while job = redis.lpop('queue:process_commit')
+        payload = JSON.load(job)
+        project = Project.find_including_path(payload['args'][0])
+
+        next unless project
+
+        begin
+          commit = project.repository.lookup(payload['args'][2])
+        rescue Rugged::OdbError
+          next
+        end
+
+        hash = {
+          id: commit.oid,
+          message: commit.message,
+          parent_ids: commit.parent_ids,
+          authored_date: commit.author[:time],
+          author_name: commit.author[:name],
+          author_email: commit.author[:email],
+          committed_date: commit.committer[:time],
+          committer_email: commit.committer[:email],
+          committer_name: commit.committer[:name]
+        }
+
+        payload['args'][2] = hash
+
+        new_jobs << JSON.dump(payload)
+      end
+
+      redis.multi do |multi|
+        new_jobs.each do |j|
+          multi.lpush('queue:process_commit', j)
+        end
+      end
+    end
+  end
+
+  def down
+    Sidekiq.redis do |redis|
+      new_jobs = []
+
+      while job = redis.lpop('queue:process_commit')
+        payload = JSON.load(job)
+
+        payload['args'][2] = payload['args'][2]['id']
+
+        new_jobs << JSON.dump(payload)
+      end
+
+      redis.multi do |multi|
+        new_jobs.each do |j|
+          multi.lpush('queue:process_commit', j)
+        end
+      end
+    end
+  end
+end