diff options
Diffstat (limited to 'lib/gitlab/git')
| -rw-r--r-- | lib/gitlab/git/blob.rb | 61 | ||||
| -rw-r--r-- | lib/gitlab/git/commit.rb | 52 | ||||
| -rw-r--r-- | lib/gitlab/git/conflict/resolver.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git/gitlab_projects.rb | 94 | ||||
| -rw-r--r-- | lib/gitlab/git/index.rb | 12 | ||||
| -rw-r--r-- | lib/gitlab/git/operation_service.rb | 5 | ||||
| -rw-r--r-- | lib/gitlab/git/ref.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/git/remote_mirror.rb | 24 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 361 | ||||
| -rw-r--r-- | lib/gitlab/git/rev_list.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/forked_storage_check.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/git/wiki_page.rb | 3 |
12 files changed, 437 insertions, 184 deletions
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 228d97a87ab..81e46028752 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -34,7 +34,7 @@ module Gitlab def raw(repository, sha) Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) end @@ -50,10 +50,19 @@ module Gitlab # to the caller to limit the number of blobs and blob_size_limit. # # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 - def batch(repository, blob_references, blob_size_limit: nil) - blob_size_limit ||= MAX_DATA_DISPLAY_SIZE - blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| + if is_enabled + Gitlab::GitalyClient.allow_n_plus_1_calls do + blob_references.map do |sha, path| + find_by_gitaly(repository, sha, path, limit: blob_size_limit) + end + end + else + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end end end @@ -61,11 +70,19 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - blob_ids.lazy - .select { |sha| possible_lfs_blob?(repository, sha) } - .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } - .select(&:lfs_pointer?) - .force + return [] if blob_ids.empty? + + repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled| + if is_enabled + repository.gitaly_blob_client.batch_lfs_pointers(blob_ids) + else + blob_ids.lazy + .select { |sha| possible_lfs_blob?(repository, sha) } + .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } + .select(&:lfs_pointer?) + .force + end + end end def binary?(data) @@ -122,13 +139,25 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path) + def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) return unless entry + entry.data = "" if limit == 0 + case entry.type when :COMMIT new( @@ -154,8 +183,10 @@ module Gitlab end def find_by_rugged(repository, sha, path, limit:) - commit = repository.lookup(sha) - root_tree = commit.tree + return unless path + + rugged_commit = repository.lookup(sha) + root_tree = rugged_commit.tree blob_entry = find_entry_by_path(repository, root_tree.oid, path) @@ -235,7 +266,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| @data = begin if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data + repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else repository.lookup(id).content end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 145721dea76..768617e2cae 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -15,8 +15,6 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator - delegate :tree, to: :rugged_commit - def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -241,6 +239,24 @@ module Gitlab end end end + + def extract_signature(repository, commit_id) + repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| + if is_enabled + repository.gitaly_commit_client.extract_signature(commit_id) + else + rugged_extract_signature(repository, commit_id) + end + end + end + + def rugged_extract_signature(repository, commit_id) + begin + Rugged::Commit.extract_signature(repository.rugged, commit_id) + rescue Rugged::OdbError + nil + end + end end def initialize(repository, raw_commit, head = nil) @@ -438,6 +454,16 @@ module Gitlab parent_ids.size > 1 end + def tree_entry(path) + @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| + if is_migrated + gitaly_tree_entry(path) + else + rugged_tree_entry(path) + end + end + end + def to_gitaly_commit return raw_commit if raw_commit.is_a?(Gitaly::GitCommit) @@ -498,6 +524,28 @@ module Gitlab SERIALIZE_KEYS end + def gitaly_tree_entry(path) + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + # To be compatible with the rugged format + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry + end + + # Is this the same as Blob.find_entry_by_path ? + def rugged_tree_entry(path) + rugged_commit.tree.path(path) + rescue Rugged::TreeError + nil + end + def gitaly_commit_author_from_rugged(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 74c9874d590..07b7e811a34 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -15,7 +15,7 @@ module Gitlab @conflicts ||= begin @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| if is_enabled - gitaly_conflicts_client(@target_repository).list_conflict_files + gitaly_conflicts_client(@target_repository).list_conflict_files.to_a else rugged_list_conflict_files end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index cba638c06db..e5a747cb987 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -41,62 +41,16 @@ module Gitlab io.read end - def rm_project - logger.info "Removing repository <#{repository_absolute_path}>." - FileUtils.rm_rf(repository_absolute_path) - end - - # Move repository from one directory to another - # - # Example: gitlab/gitlab-ci.git -> randx/six.git - # - # Won't work if target namespace directory does not exist - # - def mv_project(new_path) - new_absolute_path = File.join(shard_path, new_path) - - # verify that the source repo exists - unless File.exist?(repository_absolute_path) - logger.error "mv-project failed: source path <#{repository_absolute_path}> does not exist." - return false - end - - # ...and that the target repo does not exist - if File.exist?(new_absolute_path) - logger.error "mv-project failed: destination path <#{new_absolute_path}> already exists." - return false - end - - logger.info "Moving repository from <#{repository_absolute_path}> to <#{new_absolute_path}>." - FileUtils.mv(repository_absolute_path, new_absolute_path) - end - # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false + Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled| + if is_enabled + gitaly_import_repository(source) + else + git_import_repository(source, timeout) + end end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true end def fork_repository(new_shard_path, new_repository_relative_path) @@ -261,6 +215,42 @@ module Gitlab raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") end + def git_import_repository(source, timeout) + # Skip import if repo already exists + return false if File.exist?(repository_absolute_path) + + masked_source = mask_password_in_url(source) + + logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." + cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) + + success = run_with_timeout(cmd, timeout, nil) + + unless success + logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") + FileUtils.rm_rf(repository_absolute_path) + return false + end + + Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) + + # The project was imported successfully. + # Remove the origin URL since it may contain password. + remove_origin_in_repo + + true + end + + def gitaly_import_repository(source) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output << e.message + false + end + def git_fork_repository(new_shard_path, new_repository_relative_path) from_path = repository_absolute_path to_path = File.join(new_shard_path, new_repository_relative_path) diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index db532600d1b..d94082a3e30 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -10,6 +10,7 @@ module Gitlab DEFAULT_MODE = 0o100644 ACTIONS = %w(create create_dir update move delete).freeze + ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze attr_reader :repository, :raw_index @@ -20,6 +21,11 @@ module Gitlab delegate :read_tree, :get, to: :raw_index + def apply(action, options) + validate_action!(action) + public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend + end + def write_tree raw_index.write_tree(repository.rugged) end @@ -140,6 +146,12 @@ module Gitlab rescue Rugged::IndexError => e raise IndexError, e.message end + + def validate_action!(action) + unless ACTIONS.include?(action.to_s) + raise ArgumentError, "Unknown action '#{action}'" + end + end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ef5bdbaf819..3fb0e2eed93 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -97,6 +97,11 @@ module Gitlab end end + def update_branch(branch_name, newrev, oldrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + end + private # Returns [newrev, should_run_after_create, should_run_after_create_branch] diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 372ce005b94..a3ba9475ad0 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -33,9 +33,9 @@ module Gitlab object end - def initialize(repository, name, target, derefenced_target) + def initialize(repository, name, target, dereferenced_target) @name = Gitlab::Git.ref_name(name) - @dereferenced_target = derefenced_target + @dereferenced_target = dereferenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index 38e9d2a8554..ebe46722890 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -6,7 +6,23 @@ module Gitlab @ref_name = ref_name end - def update(only_branches_matching: [], only_tags_matching: []) + def update(only_branches_matching: []) + @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| + if is_enabled + gitaly_update(only_branches_matching) + else + rugged_update(only_branches_matching) + end + end + end + + private + + def gitaly_update(only_branches_matching) + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) + end + + def rugged_update(only_branches_matching) local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) @@ -15,8 +31,8 @@ module Gitlab delete_refs(local_branches, remote_branches) - local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching) - remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching) + local_tags = refs_obj(@repository.tags) + remote_tags = refs_obj(@repository.remote_tags(@ref_name)) updated_tags = changed_refs(local_tags, remote_tags) @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? @@ -24,8 +40,6 @@ module Gitlab delete_refs(local_tags, remote_tags) end - private - def refs_obj(refs, only_refs_matching: []) refs.each_with_object({}) do |ref, refs| next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index aec85f971ca..d7c712e75c5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -490,19 +490,17 @@ module Gitlab return [] end - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end + log_by_shell(sha, options) end def count_commits(options) + count_commits_options = process_count_commits_options(options) + gitaly_migrate(:count_commits) do |is_enabled| if is_enabled - count_commits_by_gitaly(options) + count_commits_by_gitaly(count_commits_options) else - count_commits_by_shelling_out(options) + count_commits_by_shelling_out(count_commits_options) end end end @@ -540,8 +538,8 @@ module Gitlab end # Counts the amount of commits between `from` and `to`. - def count_commits_between(from, to) - count_commits(ref: "#{from}..#{to}") + def count_commits_between(from, to, options = {}) + count_commits(from: from, to: to, **options) end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -569,7 +567,21 @@ module Gitlab end def merged_branch_names(branch_names = []) - Set.new(git_merged_branch_names(branch_names)) + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + branches = gitaly_migrate(:merged_branch_names) do |is_enabled| + if is_enabled + gitaly_merged_branch_names(branch_names, root_sha) + else + git_merged_branch_names(branch_names, root_sha) + end + end + + Set.new(branches) end # Return an array of Diff objects that represent the diff @@ -605,37 +617,6 @@ module Gitlab end end - # Returns branch names collection that contains the special commit(SHA1 - # or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branch_names_contains(commit) - branches_contains(commit).map { |c| c.name } - end - - # Returns branch collection that contains the special commit(SHA1 or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branches_contains(commit) - commit_obj = rugged.rev_parse(commit) - parent = commit_obj.parents.first unless commit_obj.parents.empty? - - walker = Rugged::Walker.new(rugged) - - rugged.branches.select do |branch| - walker.push(branch.target_id) - walker.hide(parent) if parent - result = walker.any? { |c| c.oid == commit_obj.oid } - walker.reset - - result - end - end - # Get refs hash which key is SHA1 # and value is a Rugged::Reference def refs_hash @@ -652,6 +633,7 @@ module Gitlab end end end + @refs_hash end @@ -1101,12 +1083,12 @@ module Gitlab end end - def write_ref(ref_path, ref) - raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') - raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - - input = "update #{ref_path}\x00#{ref}\x00\x00" - run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } + def write_ref(ref_path, ref, old_ref: nil, shell: true) + if shell + shell_write_ref(ref_path, ref, old_ref) + else + rugged_write_ref(ref_path, ref) + end end def fetch_ref(source_repository, source_ref:, target_ref:) @@ -1161,23 +1143,13 @@ module Gitlab end def fetch_repository_as_mirror(repository) - remote_name = "tmp-#{SecureRandom.hex}" - - # Notice that this feature flag is not for `fetch_repository_as_mirror` - # as a whole but for the fetching mechanism (file path or gitaly-ssh). - url, env = gitaly_migrate(:fetch_internal) do |is_enabled| + gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled| if is_enabled - repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) - [GITALY_INTERNAL_URL, repository.fetch_env] + gitaly_remote_client.fetch_internal_remote(repository) else - [repository.path, nil] + rugged_fetch_repository_as_mirror(repository) end end - - add_remote(remote_name, url, mirror_refmap: :all_refs) - fetch_remote(remote_name, env: env) - ensure - remove_remote(remote_name) end def blob_at(sha, path) @@ -1185,7 +1157,7 @@ module Gitlab end # Items should be of format [[commit_id, path], [commit_id1, path1]] - def batch_blobs(items, blob_size_limit: nil) + def batch_blobs(items, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) end @@ -1216,26 +1188,31 @@ module Gitlab end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repository.path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) - - rebase_sha + gitaly_migrate(:rebase) do |is_enabled| + if is_enabled + gitaly_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + else + git_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end end end def rebase_in_progress?(rebase_id) - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + gitaly_migrate(:rebase_in_progress) do |is_enabled| + if is_enabled + gitaly_repository_client.rebase_in_progress?(rebase_id) + else + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + end end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) @@ -1291,6 +1268,41 @@ module Gitlab success || gitlab_projects_error end + def bundle_to_disk(save_path) + gitaly_migrate(:bundle_to_disk) do |is_enabled| + if is_enabled + gitaly_repository_client.create_bundle(save_path) + else + run_git!(%W(bundle create #{save_path} --all)) + end + end + + true + end + + # rubocop:disable Metrics/ParameterLists + def multi_action( + user, branch_name:, message:, actions:, + author_email: nil, author_name: nil, + start_branch_name: nil, start_repository: self) + + gitaly_migrate(:operation_user_commit_files) do |is_enabled| + if is_enabled + gitaly_operation_client.user_commit_files(user, branch_name, + message, actions, author_email, author_name, + start_branch_name, start_repository) + else + rugged_multi_action(user, branch_name, message, actions, + author_email, author_name, start_branch_name, start_repository) + end + end + end + # rubocop:enable Metrics/ParameterLists + + def write_config(full_path:) + rugged.config['gitlab.fullpath'] = full_path if full_path.present? + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1319,6 +1331,10 @@ module Gitlab @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) end + def gitaly_blob_client + @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self) + end + def gitaly_conflicts_client(our_commit_oid, their_commit_oid) Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) end @@ -1335,6 +1351,25 @@ module Gitlab private + def shell_write_ref(ref_path, ref, old_ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") + + input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" + run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } + end + + def rugged_write_ref(ref_path, ref) + rugged.references.create(ref_path, ref, force: true) + rescue Rugged::ReferenceError => ex + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + rescue Rugged::OSError => ex + raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ + + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + end + def fresh_worktree?(path) File.exist?(path) && !clean_stuck_worktree(path) end @@ -1440,14 +1475,7 @@ module Gitlab sort_branches(branches, sort_by) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 - def git_merged_branch_names(branch_names = []) - return [] unless root_ref - - root_sha = find_branch(root_ref)&.target - - return [] unless root_sha - + def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} --format=%(refname:short)\ %(objectname)] + branch_names @@ -1461,27 +1489,34 @@ module Gitlab end end - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] + def gitaly_merged_branch_names(branch_names, root_sha) + qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } + + gitaly_ref_client.merged_branches(qualified_branch_names) + .reject { |b| b.target == root_sha } + .map(&:name) end - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a + def process_count_commits_options(options) + if options[:from] || options[:to] + ref = + if options[:left_right] # Compare with merge-base for left-right + "#{options[:from]}...#{options[:to]}" + else + "#{options[:from]}..#{options[:to]}" + end + + options.merge(ref: ref) + + elsif options[:ref] && options[:left_right] + from, to = options[:ref].match(/\A([^\.]*)\.{2,3}([^\.]*)\z/)[1..2] + + options.merge(from: from, to: to) + else + options + end end - # Gitaly note: JV: although #log_by_shell shells out to Git I think the - # complexity is such that we should migrate it as Ruby before trying to - # do it in Go. def log_by_shell(sha, options) limit = options[:limit].to_i offset = options[:offset].to_i @@ -1683,20 +1718,59 @@ module Gitlab end def count_commits_by_gitaly(options) - gitaly_commit_client.commit_count(options[:ref], options) + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] + else + gitaly_commit_client.commit_count(options[:ref], options) + end end def count_commits_by_shelling_out(options) + cmd = count_commits_shelling_command(options) + + raw_output = IO.popen(cmd) { |io| io.read } + + process_count_commits_raw_output(raw_output, options) + end + + def count_commits_shelling_command(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] cmd << "--max-count=#{options[:max_count]}" if options[:max_count] + cmd << "--left-right" if options[:left_right] cmd += %W[--count #{options[:ref]}] cmd += %W[-- #{options[:path]}] if options[:path].present? + cmd + end - raw_output = IO.popen(cmd) { |io| io.read } + def process_count_commits_raw_output(raw_output, options) + if options[:left_right] + result = raw_output.scan(/\d+/).map(&:to_i) - raw_output.to_i + if result.sum != options[:max_count] + result + else # Reaching max count, right is not accurate + right_option = + process_count_commits_options(options + .except(:left_right, :from, :to) + .merge(ref: options[:to])) + + right = count_commits_by_shelling_out(right_option) + + [result.first, right] # left should be accurate in the first call + end + else + raw_output.to_i + end end def gitaly_ls_files(ref) @@ -1916,6 +1990,40 @@ module Gitlab tree_id end + def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end + + def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + if remote_repository.is_a?(RemoteRepository) + env.merge!(remote_repository.fetch_env) + remote_repo_path = GITALY_INTERNAL_URL + else + remote_repo_path = remote_repository.path + end + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repo_path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -1966,6 +2074,49 @@ module Gitlab false end + def rugged_fetch_repository_as_mirror(repository) + remote_name = "tmp-#{SecureRandom.hex}" + repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) + + add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) + fetch_remote(remote_name, env: repository.fetch_env) + ensure + remove_remote(remote_name) + end + + def rugged_multi_action( + user, branch_name, message, actions, author_email, author_name, + start_branch_name, start_repository) + + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + index = Gitlab::Git::Index.new(self) + parents = [] + + if start_commit + index.read_tree(start_commit.rugged_commit.tree) + parents = [start_commit.sha] + end + + actions.each { |opts| index.apply(opts.delete(:action), opts) } + + committer = user_to_committer(user) + author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer + options = { + tree: index.write_tree, + message: message, + parents: parents, + author: author, + committer: committer + } + + create_commit(options) + end + end + def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 4974205b8fd..f8b2e7e0e21 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -95,7 +95,7 @@ module Gitlab object_output.map do |output_line| sha, path = output_line.split(' ', 2) - next if require_path && path.blank? + next if require_path && path.to_s.empty? sha end.reject(&:nil?) diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 1307f400700..0a4e557b59b 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -27,6 +27,7 @@ module Gitlab status = nil while status.nil? + if deadline > Time.now.utc sleep(wait_time) _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index a06bac4414f..669ae11a423 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -1,7 +1,7 @@ module Gitlab module Git class WikiPage - attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data # This class is meant to be serializable so that it can be constructed # by Gitaly and sent over the network to GitLab. @@ -21,6 +21,7 @@ module Gitlab @raw_data = gollum_page.raw_data @name = gollum_page.name @historical = gollum_page.historical? + @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page) @version = version end |