diff options
Diffstat (limited to 'lib/gitlab/git')
| -rw-r--r-- | lib/gitlab/git/committer_with_hooks.rb | 47 | ||||
| -rw-r--r-- | lib/gitlab/git/diff.rb | 76 | ||||
| -rw-r--r-- | lib/gitlab/git/diff_collection.rb | 5 | ||||
| -rw-r--r-- | lib/gitlab/git/gitlab_projects.rb | 253 | ||||
| -rw-r--r-- | lib/gitlab/git/hook.rb | 108 | ||||
| -rw-r--r-- | lib/gitlab/git/hooks_service.rb | 35 | ||||
| -rw-r--r-- | lib/gitlab/git/index.rb | 150 | ||||
| -rw-r--r-- | lib/gitlab/git/operation_service.rb | 173 | ||||
| -rw-r--r-- | lib/gitlab/git/popen.rb | 112 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 217 | ||||
| -rw-r--r-- | lib/gitlab/git/tree.rb | 45 | ||||
| -rw-r--r-- | lib/gitlab/git/version.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git/wiki.rb | 14 |
13 files changed, 33 insertions, 1204 deletions
diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb deleted file mode 100644 index 4198be7c9c9..00000000000 --- a/lib/gitlab/git/committer_with_hooks.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Git - class CommitterWithHooks < Gollum::Committer - attr_reader :gl_wiki - - def initialize(gl_wiki, options = {}) - @gl_wiki = gl_wiki - super(gl_wiki.gollum_wiki, options) - end - - def commit - # TODO: Remove after 10.8 - return super unless allowed_to_run_hooks? - - result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch( - @wiki.ref, - start_branch_name: @wiki.ref - ) do |start_commit| - super(false) - end - - result[:newrev] - rescue Gitlab::Git::PreReceiveError => e - message = "Custom Hook failed: #{e.message}" - raise Gitlab::Git::Wiki::OperationError, message - end - - private - - # TODO: Remove after 10.8 - def allowed_to_run_hooks? - @options[:user_id] != 0 && @options[:username].present? - end - - def git_user - @git_user ||= Gitlab::Git::User.new(@options[:username], - @options[:name], - @options[:email], - gitlab_id) - end - - def gitlab_id - Gitlab::GlId.gl_id_from_id_value(@options[:user_id]) - end - end - end -end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 61ce10ca131..f6b51dc3982 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,6 +1,3 @@ -# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. - -# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git class Diff @@ -52,20 +49,31 @@ module Gitlab repo.diff(common_commit, head, actual_options, *paths) end - # Return a copy of the +options+ hash containing only keys that can be - # passed to Rugged. Allowed options are: + # Return a copy of the +options+ hash containing only recognized keys. + # Allowed options are: # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :disable_pathspec_match :: - # If true, the given +*paths+ will be applied as exact matches, - # instead of as fnmatch patterns. + # :max_files :: + # Limit how many files will patches be allowed for before collapsing + # + # :max_lines :: + # Limit how many patch lines (across all files) will be allowed for + # before collapsing # + # :limits :: + # A hash with additional limits to check before collapsing patches. + # Allowed keys are: `max_bytes`, `safe_max_files`, `safe_max_lines` + # and `safe_max_bytes` + # + # :expanded :: + # If true, patch raw data will not be included in the diff after + # `max_files`, `max_lines` or any of the limits in `limits` are + # exceeded def filter_diff_options(options, default_options = {}) - allowed_options = [:ignore_whitespace_change, - :disable_pathspec_match, :paths, - :max_files, :max_lines, :limits, :expanded] + allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, + :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -93,7 +101,7 @@ module Gitlab # # "Binary files a/file/path and b/file/path differ\n" # This is used when we detect that a diff is binary - # using CharlockHolmes when Rugged treats it as text. + # using CharlockHolmes. def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" end @@ -106,8 +114,6 @@ module Gitlab when Hash init_from_hash(raw_diff) prune_diff_if_eligible - when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff) when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) prune_diff_if_eligible @@ -184,31 +190,6 @@ module Gitlab private - def init_from_rugged(rugged) - if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged) - d = rugged.delta - else - d = rugged - end - - @new_path = encode!(d.new_file[:path]) - @old_path = encode!(d.old_file[:path]) - @a_mode = d.old_file[:mode].to_s(8) - @b_mode = d.new_file[:mode].to_s(8) - @new_file = d.added? - @renamed_file = d.renamed? - @deleted_file = d.deleted? - end - - def init_from_rugged_patch(patch) - # Don't bother initializing diffs that are too large. If a diff is - # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch) - - @diff = encode!(strip_diff_headers(patch.to_s)) - end - def init_from_hash(hash) raw_diff = hash.symbolize_keys @@ -262,23 +243,6 @@ module Gitlab false end - - # Strip out the information at the beginning of the patch's text to match - # Grit's output - def strip_diff_headers(diff_text) - # Delete everything up to the first line that starts with '---' or - # 'Binary' - diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') - - if diff_text.start_with?('---', 'Binary') - diff_text - else - # If the diff_text did not contain a line starting with '---' or - # 'Binary', return the empty string. No idea why; we are just - # preserving behavior from before the refactor. - '' - end - end end end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 219c69893ad..20dce8d0e06 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -11,7 +11,7 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.collection_limits(options = {}) + def self.limits(options = {}) limits = {} limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) @@ -19,13 +19,14 @@ module Gitlab limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + limits[:max_patch_bytes] = Gitlab::Git::Diff::SIZE_LIMIT OpenStruct.new(limits) end def initialize(iterator, options = {}) @iterator = iterator - @limits = self.class.collection_limits(options) + @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb deleted file mode 100644 index 5ff15a787f0..00000000000 --- a/lib/gitlab/git/gitlab_projects.rb +++ /dev/null @@ -1,253 +0,0 @@ -module Gitlab - module Git - class GitlabProjects - include Gitlab::Git::Popen - include Gitlab::Utils::StrongMemoize - - # Name of shard where repositories are stored. - # Example: nfs-file06 - attr_reader :shard_name - - # Relative path is a directory name for repository with .git at the end. - # Example: gitlab-org/gitlab-test.git - attr_reader :repository_relative_path - - # This is the path at which the gitlab-shell hooks directory can be found. - # It's essential for integration between git and GitLab proper. All new - # repositories should have their hooks directory symlinked here. - attr_reader :global_hooks_path - - attr_reader :logger - - def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) - @shard_name = shard_name - @repository_relative_path = repository_relative_path - - @logger = logger - @global_hooks_path = global_hooks_path - @output = StringIO.new - end - - def output - io = @output.dup - io.rewind - io.read - end - - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - # Probably will be removed when we fully migrate to Gitaly, part of - # https://gitlab.com/gitlab-org/gitaly/issues/1124. - def repository_absolute_path - strong_memoize(:repository_absolute_path) do - File.join(shard_path, repository_relative_path) - end - end - - def shard_path - strong_memoize(:shard_path) do - Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path - end - end - - # Import project via git clone --bare - # URL must be publicly cloneable - def import_project(source, timeout) - git_import_repository(source, timeout) - end - - def fork_repository(new_shard_name, new_repository_relative_path) - git_fork_repository(new_shard_name, new_repository_relative_path) - end - - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = fetch_remote_command(name, tags, prune, force) - - setup_ssh_auth(ssh_key, known_hosts) do |env| - run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." - end - end - end - end - - def push_branches(remote_name, timeout, force, branch_names) - logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push) - cmd << '--force' if force - cmd += %W(-- #{remote_name}).concat(branch_names) - - success = run_with_timeout(cmd, timeout, repository_absolute_path) - - unless success - logger.error("Pushing branches to remote #{remote_name} failed.") - end - - success - end - - def delete_remote_branches(remote_name, branch_names) - branches = branch_names.map { |branch_name| ":#{branch_name}" } - - logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) - - success = run(cmd, repository_absolute_path) - - unless success - logger.error("Pushing deleted branches to remote #{remote_name} failed.") - end - - success - end - - protected - - def run(*args) - output, exitstatus = popen(*args) - @output << output - - exitstatus&.zero? - end - - def run_with_timeout(*args) - output, exitstatus = popen_with_timeout(*args) - @output << output - - exitstatus&.zero? - rescue Timeout::Error - @output.puts('Timed out') - - false - end - - def mask_password_in_url(url) - result = URI(url) - result.password = "*****" unless result.password.nil? - result.user = "*****" unless result.user.nil? # it's needed for oauth access_token - result - rescue - url - end - - def remove_origin_in_repo - cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) - run(cmd, repository_absolute_path) - end - - # Builds a small shell script that can be used to execute SSH with a set of - # custom options. - # - # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret - # paths with spaces in them. We trust the user not to embed single or double - # quotes in the key or value. - def custom_ssh_script(options = {}) - args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ') - - [ - "#!/bin/sh", - "exec ssh #{args} \"$@\"" - ].join("\n") - end - - # Known hosts data and private keys can be passed to gitlab-shell in the - # environment. If present, this method puts them into temporary files, writes - # a script that can substitute as `ssh`, setting the options to respect those - # files, and yields: { "GIT_SSH" => "/tmp/myScript" } - def setup_ssh_auth(key, known_hosts) - options = {} - - if key - key_file = Tempfile.new('gitlab-shell-key-file') - key_file.chmod(0o400) - key_file.write(key) - key_file.close - - options['IdentityFile'] = key_file.path - options['IdentitiesOnly'] = 'yes' - end - - if known_hosts - known_hosts_file = Tempfile.new('gitlab-shell-known-hosts') - known_hosts_file.chmod(0o400) - known_hosts_file.write(known_hosts) - known_hosts_file.close - - options['StrictHostKeyChecking'] = 'yes' - options['UserKnownHostsFile'] = known_hosts_file.path - end - - return yield({}) if options.empty? - - script = Tempfile.new('gitlab-shell-ssh-wrapper') - script.chmod(0o755) - script.write(custom_ssh_script(options)) - script.close - - yield('GIT_SSH' => script.path) - ensure - key_file&.close! - known_hosts_file&.close! - script&.close! - end - - private - - def fetch_remote_command(name, tags, prune, force) - %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| - cmd << '--prune' if prune - cmd << '--force' if force - cmd << (tags ? '--tags' : '--no-tags') - end - end - - def git_import_repository(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false - end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true - end - - def git_fork_repository(new_shard_name, new_repository_relative_path) - from_path = repository_absolute_path - new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false - end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) - end - end - end -end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb deleted file mode 100644 index 94ff5b4980a..00000000000 --- a/lib/gitlab/git/hook.rb +++ /dev/null @@ -1,108 +0,0 @@ -# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in -# app/services. We shouldn't bother migrating this until we know how -# Gitlab::Git::HooksService will be migrated. - -module Gitlab - module Git - class Hook - GL_PROTOCOL = 'web'.freeze - attr_reader :name, :path, :repository - - def initialize(name, repository) - @name = name - @repository = repository - @path = File.join(repo_path, 'hooks', name) - end - - def repo_path - repository.path - end - - def exists? - File.exist?(path) - end - - def trigger(gl_id, gl_username, oldrev, newrev, ref) - return [true, nil] unless exists? - - Bundler.with_clean_env do - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - end - end - end - - private - - def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - changes = [oldrev, newrev, ref].join(" ") - - exit_status = false - exit_message = nil - - vars = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path, - 'GL_PROTOCOL' => GL_PROTOCOL, - 'GL_REPOSITORY' => repository.gl_repository - } - - options = { - chdir: repo_path - } - - Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| - exit_status = true - stdin.sync = true - - # in git, pre- and post- receive hooks may just exit without - # reading stdin. We catch the exception to avoid a broken pipe - # warning - begin - # inject all the changes as stdin to the hook - changes.lines do |line| - stdin.puts line - end - rescue Errno::EPIPE - end - - stdin.close - - unless wait_thr.value == 0 - exit_status = false - exit_message = retrieve_error_message(stderr, stdout) - end - end - - [exit_status, exit_message] - end - - def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path - } - - options = { - chdir: repo_path - } - - args = [ref, oldrev, newrev] - - stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, stderr.presence || stdout] - end - - def retrieve_error_message(stderr, stdout) - err_message = stderr.read - err_message = err_message.blank? ? stdout.read : err_message - err_message - end - end - end -end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb deleted file mode 100644 index e67cacdb95a..00000000000 --- a/lib/gitlab/git/hooks_service.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module Git - class HooksService - attr_accessor :oldrev, :newrev, :ref - - def execute(pusher, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = pusher.gl_id - @gl_username = pusher.username - @oldrev = oldrev - @newrev = newrev - @ref = ref - - %w(pre-receive update).each do |hook_name| - status, message = run_hook(hook_name) - - unless status - raise PreReceiveError, message - end - end - - yield(self).tap do - run_hook('post-receive') - end - end - - private - - def run_hook(name) - hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) - end - end - end -end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index d94082a3e30..c2e4274e3ee 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,157 +1,7 @@ -# Gitaly note: JV: When the time comes I think we will want to copy this -# class into Gitaly. None of its methods look like they should be RPC's. -# The RPC's will be at a higher level. - module Gitlab module Git class Index IndexError = Class.new(StandardError) - - DEFAULT_MODE = 0o100644 - - ACTIONS = %w(create create_dir update move delete).freeze - ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze - - attr_reader :repository, :raw_index - - def initialize(repository) - @repository = repository - @raw_index = repository.rugged.index - end - - delegate :read_tree, :get, to: :raw_index - - def apply(action, options) - validate_action!(action) - public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend - end - - def write_tree - raw_index.write_tree(repository.rugged) - end - - def dir_exists?(path) - raw_index.find { |entry| entry[:path].start_with?("#{path}/") } - end - - def create(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - add_blob(options) - end - - def create_dir(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - if dir_exists?(options[:file_path]) - raise IndexError, "A directory with this name already exists" - end - - options = options.dup - options[:file_path] += '/.gitkeep' - options[:content] = '' - - add_blob(options) - end - - def update(options) - options = normalize_options(options) - - file_entry = get(options[:file_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - add_blob(options, mode: file_entry[:mode]) - end - - def move(options) - options = normalize_options(options) - - file_entry = get(options[:previous_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - raw_index.remove(options[:previous_path]) - - add_blob(options, mode: file_entry[:mode]) - end - - def delete(options) - options = normalize_options(options) - - unless get(options[:file_path]) - raise IndexError, "A file with this name doesn't exist" - end - - raw_index.remove(options[:file_path]) - end - - private - - def normalize_options(options) - options = options.dup - options[:file_path] = normalize_path(options[:file_path]) if options[:file_path] - options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path] - options - end - - def normalize_path(path) - unless path - raise IndexError, "You must provide a file path" - end - - pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - - pathname.each_filename do |segment| - if segment == '..' - raise IndexError, 'Path cannot include directory traversal' - end - end - - pathname.to_s - end - - def add_blob(options, mode: nil) - content = options[:content] - unless content - raise IndexError, "You must provide content" - end - - content = Base64.decode64(content) if options[:encoding] == 'base64' - - detect = CharlockHolmes::EncodingDetector.new.detect(content) - unless detect && detect[:type] == :binary - # When writing to the repo directly as we are doing here, - # the `core.autocrlf` config isn't taken into account. - content.gsub!("\r\n", "\n") if repository.autocrlf - end - - oid = repository.rugged.write(content, :blob) - - raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) - rescue Rugged::IndexError => e - raise IndexError, e.message - end - - def validate_action!(action) - unless ACTIONS.include?(action.to_s) - raise ArgumentError, "Unknown action '#{action}'" - end - end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 57d748343be..0584629ac84 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,8 +1,6 @@ module Gitlab module Git class OperationService - include Gitlab::Git::Popen - BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created @@ -17,177 +15,6 @@ module Gitlab ) end end - - attr_reader :user, :repository - - def initialize(user, new_repository) - if user - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - @user = user - end - - # Refactoring aid - Gitlab::Git.check_namespace!(new_repository) - - @repository = new_repository - end - - def add_branch(branch_name, newrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - oldrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def rm_branch(branch) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name - oldrev = branch.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def add_tag(tag_name, newrev, options = {}) - ref = Gitlab::Git::TAG_REF_PREFIX + tag_name - oldrev = Gitlab::Git::BLANK_SHA - - with_hooks(ref, newrev, oldrev) do |service| - # We want to pass the OID of the tag object to the hooks. For an - # annotated tag we don't know that OID until after the tag object - # (raw_tag) is created in the repository. That is why we have to - # update the value after creating the tag object. Only the - # "post-receive" hook will receive the correct value in this case. - raw_tag = repository.rugged.tags.create(tag_name, newrev, options) - service.newrev = raw_tag.target_id - end - end - - def rm_tag(tag) - ref = Gitlab::Git::TAG_REF_PREFIX + tag.name - oldrev = tag.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) do - repository.rugged.tags.delete(tag_name) - end - end - - # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, - # it would be created from `start_branch_name`. - # If `start_repository` is passed, and the branch doesn't exist, - # it would try to find the commits from it instead of current repository. - def with_branch( - branch_name, - start_branch_name: nil, - start_repository: repository, - &block) - - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - start_branch_name = nil if start_repository.empty? - - if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" - end - - update_branch_with_hooks(branch_name) do - repository.with_repo_branch_commit( - start_repository, - start_branch_name || branch_name, - &block) - end - end - - def update_branch(branch_name, newrev, oldrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - end - - private - - # Returns [newrev, should_run_after_create, should_run_after_create_branch] - def update_branch_with_hooks(branch_name) - update_autocrlf_option - - was_empty = repository.empty? - - # Make commit - newrev = yield - - unless newrev - raise Gitlab::Git::CommitError.new('Failed to create commit') - end - - branch = repository.find_branch(branch_name) - oldrev = find_oldrev_from_branch(newrev, branch) - - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - - BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) - end - - def find_oldrev_from_branch(newrev, branch) - return Gitlab::Git::BLANK_SHA unless branch - - oldrev = branch.target - - merge_base = repository.merge_base(newrev, branch.target) - raise Gitlab::Git::Repository::InvalidRef unless merge_base - - if oldrev == merge_base - oldrev - else - raise Gitlab::Git::CommitError.new('Branch diverged') - end - end - - def update_ref_in_hooks(ref, newrev, oldrev) - with_hooks(ref, newrev, oldrev) do - update_ref(ref, newrev, oldrev) - end - end - - def with_hooks(ref, newrev, oldrev) - Gitlab::Git::HooksService.new.execute( - user, - repository, - oldrev, - newrev, - ref) do |service| - - yield(service) - end - end - - # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. - def update_ref(ref, newrev, oldrev) - # We use 'git update-ref' because libgit2/rugged currently does not - # offer 'compare and swap' ref updates. Without compare-and-swap we can - # (and have!) accidentally reset the ref to an earlier state, clobbering - # commits. See also https://github.com/libgit2/libgit2/issues/1534. - command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - - output, status = popen( - command, - repository.path) do |stdin| - stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") - end - - unless status.zero? - Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") - raise Gitlab::Git::CommitError.new( - "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ - " Please refresh and try again.") - end - end - - def update_autocrlf_option - if repository.autocrlf != :input - repository.autocrlf = :input - end - end end end end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb deleted file mode 100644 index 7426688fc55..00000000000 --- a/lib/gitlab/git/popen.rb +++ /dev/null @@ -1,112 +0,0 @@ -# Gitaly note: JV: no RPC's here. - -require 'open3' - -module Gitlab - module Git - module Popen - FAST_GIT_PROCESS_TIMEOUT = 15.seconds - - def popen(cmd, path, vars = {}, lazy_block: nil) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - options = { chdir: path } - - cmd_output = "" - cmd_status = 0 - Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - stdout.set_encoding(Encoding::ASCII_8BIT) - - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - err_reader = Thread.new { stderr.read } - - yield(stdin) if block_given? - stdin.close - - if lazy_block - cmd_output = lazy_block.call(stdout.lazy) - cmd_status = 0 - break - else - cmd_output << stdout.read - end - - cmd_output << err_reader.value - cmd_status = wait_thr.value.exitstatus - end - - [cmd_output, cmd_status] - end - - def popen_with_timeout(cmd, timeout, path, vars = {}) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - - unless File.directory?(path) - FileUtils.mkdir_p(path) - end - - rout, wout = IO.pipe - rerr, werr = IO.pipe - - pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - out_reader = Thread.new { rout.read } - err_reader = Thread.new { rerr.read } - - begin - # close write ends so we could read them - wout.close - werr.close - - status = process_wait_with_timeout(pid, timeout) - - cmd_output = out_reader.value - cmd_output << err_reader.value # Copying the behaviour of `popen` which merges stderr into output - - [cmd_output, status.exitstatus] - rescue Timeout::Error => e - kill_process_group_for_pid(pid) - - raise e - ensure - wout.close unless wout.closed? - werr.close unless werr.closed? - - rout.close - rerr.close - end - end - - def process_wait_with_timeout(pid, timeout) - deadline = timeout.seconds.from_now - wait_time = 0.01 - - while deadline > Time.now - sleep(wait_time) - _, status = Process.wait2(pid, Process::WNOHANG) - - return status unless status.nil? - end - - raise Timeout::Error, "Timeout waiting for process ##{pid}" - end - - def kill_process_group_for_pid(pid) - Process.kill("KILL", -pid) - Process.wait(pid) - rescue Errno::ESRCH - end - end - end -end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9521a2d63a0..1b8d320ff3b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,7 +6,6 @@ module Gitlab module Git class Repository include Gitlab::Git::RepositoryMirroring - include Gitlab::Git::Popen include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize @@ -73,7 +72,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -82,13 +81,6 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage, - relative_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - @name = @relative_path.split("/").last end @@ -121,10 +113,6 @@ module Gitlab raise NoRepository.new('no repository for such path') end - def cleanup - @rugged&.close - end - def circuit_breaker @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end @@ -148,10 +136,6 @@ module Gitlab end end - def reload_rugged - @rugged = nil - end - # Directly find a branch with a simple name (e.g. master) # def find_branch(name) @@ -250,15 +234,6 @@ module Gitlab end end - # Returns an Array of all ref names, except when it's matching pattern - # - # regexp - The pattern for ref names we don't want - def all_ref_names_except(prefixes) - rugged.references.reject do |ref| - prefixes.any? { |p| ref.name.start_with?(p) } - end.map(&:name) - end - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) @@ -331,7 +306,7 @@ module Gitlab (size.to_f / 1024).round(2) end - # Use the Rugged Walker API to build an array of commits. + # Build an array of commits. # # Usage. # repo.log( @@ -591,19 +566,6 @@ module Gitlab end end - def check_revert_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << { mainline: 1 } if target_commit.merge_commit? - - revert_index = rugged.revert_commit(*args) - return false if revert_index.conflicts? - - tree_id = revert_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) args = { user: user, @@ -619,10 +581,6 @@ module Gitlab end end - def diff_exists?(sha1, sha2) - rugged.diff(sha1, sha2).size > 0 - end - def user_to_committer(user) Gitlab::Git.committer_hash(email: user.email, name: user.name) end @@ -666,6 +624,14 @@ module Gitlab end end + def find_remote_root_ref(remote_name) + return unless remote_name.present? + + wrapped_gitaly_errors do + gitaly_remote_client.find_remote_root_ref(remote_name) + end + end + AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -738,48 +704,6 @@ module Gitlab end end - def with_repo_branch_commit(start_repository, start_branch_name) - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - return yield nil if start_repository.empty? - - if start_repository.same_repository?(self) - yield commit(start_branch_name) - else - start_commit_id = start_repository.commit_id(start_branch_name) - - return yield nil unless start_commit_id - - if branch_commit = commit(start_commit_id) - yield branch_commit - else - with_repo_tmp_commit( - start_repository, start_branch_name, start_commit_id) do |tmp_commit| - yield tmp_commit - end - end - end - end - - def with_repo_tmp_commit(start_repository, start_branch_name, sha) - source_ref = start_branch_name - - unless Gitlab::Git.branch_ref?(source_ref) - source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" - end - - tmp_ref = fetch_ref( - start_repository, - source_ref: source_ref, - target_ref: "refs/tmp/#{SecureRandom.hex}" - ) - - yield commit(sha) - ensure - delete_refs(tmp_ref) if tmp_ref - end - def fetch_source_branch!(source_repository, source_branch, local_ref) wrapped_gitaly_errors do gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) @@ -809,21 +733,6 @@ module Gitlab end end - # This method, fetch_ref, is used from within - # Gitlab::Git::OperationService. OperationService will eventually only - # exist in gitaly-ruby. When we delete OperationService from gitlab-ce - # we can also remove fetch_ref. - def fetch_ref(source_repository, source_ref:, target_ref:) - Gitlab::Git.check_namespace!(source_repository) - source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) - - args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) - message, status = run_git(args, env: source_repository.fetch_env) - raise Gitlab::Git::CommandError, message if status != 0 - - target_ref - end - # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) @@ -891,24 +800,6 @@ module Gitlab end end - def push_remote_branches(remote_name, branch_names, forced: true) - success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - def bundle_to_disk(save_path) wrapped_gitaly_errors do gitaly_repository_client.create_bundle(save_path) @@ -1056,37 +947,12 @@ module Gitlab end end - def shell_blame(sha, path) - output, _status = run_git(%W(blame -p #{sha} -- #{path})) - output - end - def last_commit_for_path(sha, path) wrapped_gitaly_errors do gitaly_commit_client.last_commit_for_path(sha, path) end end - def rev_list(including: [], excluding: [], options: [], objects: false, &block) - args = ['rev-list'] - - args.push(*rev_list_param(including)) - - exclude_param = *rev_list_param(excluding) - if exclude_param.any? - args.push('--not') - args.push(*exclude_param) - end - - args.push('--objects') if objects - - if options.any? - args.push(*options) - end - - run_git!(args, lazy_block: block) - end - def checksum # The exists? RPC is much cheaper, so we perform this request first raise NoRepository, "Repository does not exists" unless exists? @@ -1104,44 +970,6 @@ module Gitlab end end - def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - cmd = [Gitlab.config.git.bin_path, *args] - cmd.unshift("nice") if nice - - object_directories = alternate_object_directories - if object_directories.any? - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) - end - - circuit_breaker.perform do - popen(cmd, chdir, env, lazy_block: lazy_block, &block) - end - end - - def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) - - raise GitError, output unless status.zero? - - output - end - - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - def git_env_for_user(user) - { - 'GIT_COMMITTER_NAME' => user.name, - 'GIT_COMMITTER_EMAIL' => user.email, - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, - 'GL_REPOSITORY' => gl_repository - } - end - def gitaly_merged_branch_names(branch_names, root_sha) qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } @@ -1192,23 +1020,6 @@ module Gitlab Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact end - def sort_branches(branches, sort_by) - case sort_by - when 'name' - branches.sort_by(&:name) - when 'updated_desc' - branches.sort do |a, b| - b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date - end - when 'updated_asc' - branches.sort do |a, b| - a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date - end - else - branches - end - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -1223,14 +1034,6 @@ module Gitlab def gitaly_delete_refs(*ref_names) gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end - - def gitlab_projects_error - raise CommandError, @gitlab_projects.output - end - - def rev_list_param(spec) - spec == :all ? ['--all'] : spec - end end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index cb851b76a23..e0867aeb5a7 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -50,51 +50,6 @@ module Gitlab entry[:oid] end end - - def tree_entries_from_rugged(repository, sha, path, recursive) - current_path_entries = get_tree_entries_from_rugged(repository, sha, path) - ordered_entries = [] - - current_path_entries.each do |entry| - ordered_entries << entry - - if recursive && entry.dir? - ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) - end - end - - ordered_entries - end - - def get_tree_entries_from_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) - end - rescue Rugged::ReferenceError - [] - end end def initialize(options) diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb index 1e14e8b652a..4bd91898457 100644 --- a/lib/gitlab/git/version.rb +++ b/lib/gitlab/git/version.rb @@ -1,8 +1,6 @@ module Gitlab module Git module Version - extend Gitlab::Git::Popen - def self.git_version Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 9d992be66eb..ae92a624e05 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -163,20 +163,6 @@ module Gitlab Gitlab::Git::WikiPage.new(wiki_page, version) end end - - def committer_with_hooks(commit_details) - Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h) - end - - def with_committer_with_hooks(commit_details, &block) - committer = committer_with_hooks(commit_details) - - yield committer - - committer.commit - - nil - end end end end |