diff options
-rw-r--r-- | app/models/repository.rb | 2 | ||||
-rw-r--r-- | app/services/git_operation_service.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/git/commit.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/git/repository.rb | 164 | ||||
-rw-r--r-- | lib/gitlab/git_ref_validator.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/shell.rb | 13 | ||||
-rw-r--r-- | spec/lib/gitlab/git/repository_spec.rb | 16 |
7 files changed, 113 insertions, 87 deletions
diff --git a/app/models/repository.rb b/app/models/repository.rb index 10b429c707e..8663cf5e602 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -123,6 +123,7 @@ class Repository commits end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/384 def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0) unless exists? && has_visible_content? && query.present? return [] @@ -610,6 +611,7 @@ class Repository commit(sha) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/383 def last_commit_id_for_path(sha, path) key = path.blank? ? "last_commit_id_for_path:#{sha}" : "last_commit_id_for_path:#{sha}:#{Digest::SHA1.hexdigest(path)}" diff --git a/app/services/git_operation_service.rb b/app/services/git_operation_service.rb index 43636fde0be..32925e9c1f2 100644 --- a/app/services/git_operation_service.rb +++ b/app/services/git_operation_service.rb @@ -129,6 +129,7 @@ class GitOperationService end end + # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. def update_ref(ref, newrev, oldrev) # We use 'git update-ref' because libgit2/rugged currently does not # offer 'compare and swap' ref updates. Without compare-and-swap we can diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 8ab3b0498ff..d0f04d25db2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -38,7 +38,7 @@ module Gitlab repo = options.delete(:repo) raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) - repo.log(options).map { |c| decorate(c) } + repo.log(options) end # Get single commit diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index bd9993f275d..639f5625d59 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -331,85 +331,10 @@ module Gitlab # ) # def log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output = IO.popen(cmd) { |io| io.read } - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + raw_log(options).map { |c| Commit.decorate(c) } end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] @@ -454,7 +379,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - commits_between(from, to).size + Commit.between(self, from, to).size end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -912,6 +837,89 @@ module Gitlab private + def raw_log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_NONE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + # Gitaly note: JV: although #log_by_shell shells out to Git I think the + # complexity is such that we should migrate it as Ruby before trying to + # do it in Go. + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) + end + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + # We are trying to deprecate this method because it does a lot of work # but it seems to be used only to look up submodule URL's. # https://gitlab.com/gitlab-org/gitaly/issues/329 diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0e87ee30c98..a3c6b21a6a1 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: does not need to be migrated, works without a repo. + module Gitlab module GitRefValidator extend self diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0baea092e6a..4366ff336ef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,3 +1,6 @@ +# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. +# SSH key operations are not part of Gitaly so will never be migrated. + require 'securerandom' module Gitlab @@ -68,6 +71,7 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'add-project', storage, "#{name}.git"]) @@ -81,6 +85,7 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. @@ -99,6 +104,7 @@ module Gitlab # Ex. # fetch_remote("gitlab/gitlab-ci", "upstream") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(storage, name, remote, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced @@ -115,6 +121,7 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def mv_repository(storage, path, new_path) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', storage, "#{path}.git", "#{new_path}.git"]) @@ -129,6 +136,7 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # + # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', forked_from_storage, "#{path}.git", forked_to_storage, @@ -143,6 +151,7 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def remove_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'rm-project', storage, "#{name}.git"]) @@ -194,6 +203,7 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -207,6 +217,7 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) FileUtils.rm_r(full_path(storage, name), force: true) end @@ -216,6 +227,7 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) return false if exists?(storage, new_name) || !exists?(storage, old_name) @@ -241,6 +253,7 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) File.exist?(full_path(storage, dir_name)) end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index acffd335e43..10fa5f4044b 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -705,9 +705,9 @@ describe Gitlab::Git::Repository, seed_helper: true do # Add new commits so that there's a renamed file in the commit history repo = Gitlab::Git::Repository.new('default', TEST_REPO_PATH).rugged - commit_with_old_name = new_commit_edit_old_file(repo) - rename_commit = new_commit_move_file(repo) - commit_with_new_name = new_commit_edit_new_file(repo) + commit_with_old_name = Gitlab::Git::Commit.decorate(new_commit_edit_old_file(repo)) + rename_commit = Gitlab::Git::Commit.decorate(new_commit_move_file(repo)) + commit_with_new_name = Gitlab::Git::Commit.decorate(new_commit_edit_new_file(repo)) end after(:context) do @@ -880,8 +880,8 @@ describe Gitlab::Git::Repository, seed_helper: true do context "compare results between log_by_walk and log_by_shell" do let(:options) { { ref: "master" } } - let(:commits_by_walk) { repository.log(options).map(&:oid) } - let(:commits_by_shell) { repository.log(options.merge({ disable_walk: true })).map(&:oid) } + let(:commits_by_walk) { repository.log(options).map(&:id) } + let(:commits_by_shell) { repository.log(options.merge({ disable_walk: true })).map(&:id) } it { expect(commits_by_walk).to eq(commits_by_shell) } @@ -924,7 +924,7 @@ describe Gitlab::Git::Repository, seed_helper: true do expect(commits.size).to be > 0 expect(commits).to satisfy do |commits| - commits.all? { |commit| commit.time >= options[:after] } + commits.all? { |commit| commit.committed_date >= options[:after] } end end end @@ -937,7 +937,7 @@ describe Gitlab::Git::Repository, seed_helper: true do expect(commits.size).to be > 0 expect(commits).to satisfy do |commits| - commits.all? { |commit| commit.time <= options[:before] } + commits.all? { |commit| commit.committed_date <= options[:before] } end end end @@ -946,7 +946,7 @@ describe Gitlab::Git::Repository, seed_helper: true do let(:options) { { ref: 'master', path: ['PROCESS.md', 'README.md'] } } def commit_files(commit) - commit.diff(commit.parent_ids.first).deltas.flat_map do |delta| + commit.diff_from_parent.deltas.flat_map do |delta| [delta.old_file[:path], delta.new_file[:path]].uniq.compact end end |