diff options
Diffstat (limited to 'lib/gitlab')
| -rw-r--r-- | lib/gitlab/git/blob.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/git/commit.rb | 22 | ||||
| -rw-r--r-- | lib/gitlab/git/ref.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/git/rugged_impl/blob.rb | 106 | ||||
| -rw-r--r-- | lib/gitlab/git/rugged_impl/commit.rb | 109 | ||||
| -rw-r--r-- | lib/gitlab/git/rugged_impl/ref.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/git/rugged_impl/repository.rb | 78 | ||||
| -rw-r--r-- | lib/gitlab/git/rugged_impl/tree.rb | 109 | ||||
| -rw-r--r-- | lib/gitlab/git/tree.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/storage_settings.rb | 8 |
11 files changed, 463 insertions, 3 deletions
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 259a2b7911a..10df4ed72d9 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -23,6 +23,10 @@ module Gitlab class << self def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + tree_entry(repository, sha, path, limit) + end + + def tree_entry(repository, sha, path, limit) return unless path path = path.sub(%r{\A/*}, '') @@ -179,3 +183,5 @@ module Gitlab end end end + +Gitlab::Git::Blob.singleton_class.prepend Gitlab::Git::RuggedImpl::Blob::ClassMethods diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5863815ca85..7be8539f380 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -5,6 +5,7 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper + prepend Gitlab::Git::RuggedImpl::Commit extend Gitlab::Git::WrapsGitalyErrors attr_accessor :raw_commit, :head @@ -62,15 +63,19 @@ module Gitlab # This saves us an RPC round trip. return nil if commit_id.include?(':') - commit = wrapped_gitaly_errors do - repo.gitaly_commit_client.find_commit(commit_id) - end + commit = find_commit(repo, commit_id) decorate(repo, commit) if commit rescue Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, ArgumentError nil end + def find_commit(repo, commit_id) + wrapped_gitaly_errors do + repo.gitaly_commit_client.find_commit(commit_id) + end + end + # Get last commit for HEAD # # Ex. @@ -185,6 +190,10 @@ module Gitlab @repository = repository @head = head + init_commit(raw_commit) + end + + def init_commit(raw_commit) case raw_commit when Hash init_from_hash(raw_commit) @@ -305,11 +314,16 @@ module Gitlab def tree_entry(path) return unless path.present? + commit_tree_entry(path) + end + + def commit_tree_entry(path) # We're only interested in metadata, so limit actual data to 1 byte # since Gitaly doesn't support "send no data" option. entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) return unless entry + # To be compatible with the rugged format entry = entry.to_h entry.delete(:data) entry[:name] = File.basename(path) @@ -400,3 +414,5 @@ module Gitlab end end end + +Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index eec91194949..47cfb483509 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -4,6 +4,7 @@ module Gitlab module Git class Ref include Gitlab::EncodingHelper + include Gitlab::Git::RuggedImpl::Ref # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 54bbd531398..7516afb5ec2 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -11,6 +11,7 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize + prepend Gitlab::Git::RuggedImpl::Repository SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb new file mode 100644 index 00000000000..11ee4ebda4b --- /dev/null +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Blob + module ClassMethods + extend ::Gitlab::Utils::Override + + override :tree_entry + def tree_entry(repository, sha, path, limit) + if Feature.enabled?(:rugged_tree_entry) + rugged_tree_entry(repository, sha, path, limit) + else + super + end + end + + private + + def rugged_tree_entry(repository, sha, path, limit) + return unless path + + # Strip any leading / characters from the path + path = path.sub(%r{\A/*}, '') + + rugged_commit = repository.lookup(sha) + root_tree = rugged_commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) + + return unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + # Rugged::Blob#content is expensive; don't call it if we don't have to. + data: limit.zero? ? '' : blob.content(limit), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + rescue Rugged::ReferenceError + nil + end + + # Recursive search of blob id by path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # file.rb # oid: 4a + # + # + # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' + # + def find_entry_by_path(repository, root_id, *path_parts) + root_tree = repository.lookup(root_id) + + entry = root_tree.find do |entry| + entry[:name] == path_parts[0] + end + + return unless entry + + if path_parts.size > 1 + return unless entry[:type] == :tree + + path_parts.shift + find_entry_by_path(repository, entry[:oid], *path_parts) + else + [:blob, :commit].include?(entry[:type]) ? entry : nil + end + end + + def submodule_blob(blob_entry, path, sha) + new( + id: blob_entry[:oid], + name: blob_entry[:name], + size: 0, + data: '', + path: path, + commit_id: sha + ) + end + end + end + end + end +end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb new file mode 100644 index 00000000000..bce4fa14fb4 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Commit + module ClassMethods + extend ::Gitlab::Utils::Override + + def rugged_find(repo, commit_id) + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(::Rugged::Commit) ? obj : nil + rescue ::Rugged::Error + nil + end + + # This needs to return an array of Gitlab::Git:Commit objects + # instead of Rugged::Commit objects to ensure upstream models + # operate on a consistent interface. Unlike + # Gitlab::Git::Commit.find, Gitlab::Git::Commit.batch_by_oid + # doesn't attempt to decorate the result. + def rugged_batch_by_oid(repo, oids) + oids.map { |oid| rugged_find(repo, oid) } + .compact + .map { |commit| decorate(repo, commit) } + end + + override :find_commit + def find_commit(repo, commit_id) + if Feature.enabled?(:rugged_find_commit) + rugged_find(repo, commit_id) + else + super + end + end + + override :batch_by_oid + def batch_by_oid(repo, oids) + if Feature.enabled?(:rugged_list_commits_by_oid) + rugged_batch_by_oid(repo, oids) + else + super + end + end + end + + extend ::Gitlab::Utils::Override + + override :init_commit + def init_commit(raw_commit) + case raw_commit + when ::Rugged::Commit + init_from_rugged(raw_commit) + else + super + end + end + + override :commit_tree_entry + def commit_tree_entry(path) + if Feature.enabled?(:rugged_commit_tree_entry) + rugged_tree_entry(path) + else + super + end + end + + # Is this the same as Blob.find_entry_by_path ? + def rugged_tree_entry(path) + rugged_commit.tree.path(path) + rescue Rugged::TreeError + nil + end + + def rugged_commit + @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit) + raw_commit + else + @repository.rev_parse_target(id) + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/ref.rb b/lib/gitlab/git/rugged_impl/ref.rb new file mode 100644 index 00000000000..b553e82dc47 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/ref.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Ref + def self.dereference_object(object) + object = object.target while object.is_a?(::Rugged::Tag::Annotation) + + object + end + end + end + end +end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb new file mode 100644 index 00000000000..e91b0ddcd31 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Git + module RuggedImpl + module Repository + extend ::Gitlab::Utils::Override + + FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid).freeze + + def alternate_object_directories + relative_object_directories.map { |d| File.join(path, d) } + end + + ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY_RELATIVE + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE + ].freeze + + def relative_object_directories + Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + end + + def rugged + @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) + rescue ::Rugged::RepositoryError, ::Rugged::OSError + raise ::Gitlab::Git::Repository::NoRepository.new('no repository for such path') + end + + def cleanup + @rugged&.close + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + + override :ancestor? + def ancestor?(from, to) + if Feature.enabled?(:rugged_commit_is_ancestor) + rugged_is_ancestor?(from, to) + else + super + end + end + + def rugged_is_ancestor?(ancestor_id, descendant_id) + return false if ancestor_id.nil? || descendant_id.nil? + + rugged_merge_base(ancestor_id, descendant_id) == ancestor_id + rescue Rugged::OdbError + false + end + + def rugged_merge_base(from, to) + rugged.merge_base(from, to) + rescue Rugged::ReferenceError + nil + end + + # Lookup for rugged object by oid or ref name + def lookup(oid_or_ref_name) + rugged.rev_parse(oid_or_ref_name) + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb new file mode 100644 index 00000000000..bb13d114d46 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +# NOTE: This code is legacy. Do not add/modify code here unless you have +# discussed with the Gitaly team. See +# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code +# for more details. + +module Gitlab + module Git + module RuggedImpl + module Tree + module ClassMethods + extend ::Gitlab::Utils::Override + + override :tree_entries + def tree_entries(repository, sha, path, recursive) + if Feature.enabled?(:rugged_tree_entries) + tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) + else + super + end + end + + def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) + tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries| + # This was an optimization to reduce N+1 queries for Gitaly + # (https://gitlab.com/gitlab-org/gitaly/issues/530). It + # used to be done lazily in the view via + # TreeHelper#flatten_tree, so it's possible there's a + # performance impact by loading this eagerly. + rugged_populate_flat_path(repository, sha, path, entries) + end + end + + def tree_entries_from_rugged(repository, sha, path, recursive) + current_path_entries = get_tree_entries_from_rugged(repository, sha, path) + ordered_entries = [] + + current_path_entries.each do |entry| + ordered_entries << entry + + if recursive && entry.dir? + ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) + end + end + end + + def rugged_populate_flat_path(repository, sha, path, entries) + entries.each do |entry| + entry.flat_path = entry.path + + next unless entry.dir? + + entry.flat_path = + if path + File.join(path, rugged_flatten_tree(repository, sha, entry, path)) + else + rugged_flatten_tree(repository, sha, entry, path) + end + end + end + + # Returns the relative path of the first subdir that doesn't have only one directory descendant + def rugged_flatten_tree(repository, sha, tree, root_path) + subtree = tree_entries_from_rugged(repository, sha, tree.path, false) + + if subtree.count == 1 && subtree.first.dir? + File.join(tree.name, rugged_flatten_tree(repository, sha, subtree.first, root_path)) + else + tree.name + end + end + + def get_tree_entries_from_rugged(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + current_path = path ? File.join(path, entry[:name]) : entry[:name] + + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode].to_s(8), + path: current_path, + commit_id: sha + ) + end + rescue Rugged::ReferenceError + [] + end + end + end + end + end +end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 51542bcaaa2..33505372eaf 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -18,6 +18,10 @@ module Gitlab def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' + tree_entries(repository, sha, path, recursive) + end + + def tree_entries(repository, sha, path, recursive) wrapped_gitaly_errors do repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) end @@ -95,3 +99,5 @@ module Gitlab end end end + +Gitlab::Git::Tree.singleton_class.prepend Gitlab::Git::RuggedImpl::Tree::ClassMethods diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 754cccb6b3f..78ef6bfc0ec 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -32,11 +32,19 @@ module Gitlab end def self.disk_access_denied? + return false if rugged_enabled? + !temporarily_allowed?(ALLOW_KEY) && GitalyClient.feature_enabled?(DISK_ACCESS_DENIED_FLAG) rescue false # Err on the side of caution, don't break gitlab for people end + def self.rugged_enabled? + Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag| + Feature.enabled?(flag) + end + end + def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') |