From a4e75e7a83082c190474595ed9894ec86061d37f Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 9 Jul 2018 12:50:17 +0200 Subject: Use Gitaly for fetches and creating bundles --- lib/gitlab/git/repository.rb | 16 ++--- lib/gitlab/shell.rb | 44 +------------- spec/lib/gitlab/git/repository_spec.rb | 70 ++++++++++------------ spec/lib/gitlab/shell_spec.rb | 105 ++++++++------------------------- spec/lib/gitlab/workhorse_spec.rb | 24 ++++---- 5 files changed, 73 insertions(+), 186 deletions(-) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 420790f45d0..00c136cab41 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -898,12 +898,8 @@ module Gitlab end def fetch_source_branch!(source_repository, source_branch, local_ref) - Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| - if is_enabled - gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) - else - rugged_fetch_source_branch(source_repository, source_branch, local_ref) - end + wrapped_gitaly_errors do + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) end end @@ -1058,12 +1054,8 @@ module Gitlab end def bundle_to_disk(save_path) - gitaly_migrate(:bundle_to_disk) do |is_enabled| - if is_enabled - gitaly_repository_client.create_bundle(save_path) - else - run_git!(%W(bundle create #{save_path} --all)) - end + wrapped_gitaly_errors do + gitaly_repository_client.create_bundle(save_path) end true diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index e222541992a..a17cd27e82d 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -92,21 +92,13 @@ module Gitlab # Ex. # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) if url.start_with?('.', '/') raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end relative_path = "#{name}.git" - cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - GitalyGitlabProjects.new(storage, relative_path) - else - # The timeout ensures the subprocess won't hang forever - gitlab_projects(storage, relative_path) - end - end + cmd = GitalyGitlabProjects.new(storage, relative_path) success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success @@ -126,12 +118,8 @@ module Gitlab # fetch_remote(my_repo, "upstream") # def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - gitaly_migrate(:fetch_remote) do |is_enabled| - if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) - else - local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) - end + wrapped_gitaly_errors do + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) end end @@ -389,28 +377,6 @@ module Gitlab ) end - def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - vars = { force: forced, tags: !no_tags, prune: prune } - - if ssh_auth&.ssh_import? - if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? - vars[:ssh_key] = ssh_auth.ssh_private_key - end - - if ssh_auth.ssh_known_hosts.present? - vars[:known_hosts] = ssh_auth.ssh_known_hosts - end - end - - cmd = gitlab_projects(storage_name, repository_relative_path) - - success = cmd.fetch_remote(remote, git_timeout, vars) - - raise Error, cmd.output unless success - - success - end - def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -440,10 +406,6 @@ module Gitlab Gitlab.config.gitlab_shell.git_timeout end - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) } - end - def wrapped_gitaly_errors yield rescue GRPC::NotFound, GRPC::BadStatus => e diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index e6268a05d44..4f8e6f29255 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1716,59 +1716,51 @@ describe Gitlab::Git::Repository, seed_helper: true do end describe '#fetch_source_branch!' do - shared_examples '#fetch_source_branch!' do - let(:local_ref) { 'refs/merge-requests/1/head' } - let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') } - let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') } - - after do - ensure_seeds - end + let(:local_ref) { 'refs/merge-requests/1/head' } + let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH, '') } + let(:source_repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') } - context 'when the branch exists' do - context 'when the commit does not exist locally' do - let(:source_branch) { 'new-branch-for-fetch-source-branch' } - let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } } - let(:new_oid) { new_commit_edit_old_file(source_rugged).oid } + after do + ensure_seeds + end - before do - source_rugged.branches.create(source_branch, new_oid) - end + context 'when the branch exists' do + context 'when the commit does not exist locally' do + let(:source_branch) { 'new-branch-for-fetch-source-branch' } + let(:source_rugged) { Gitlab::GitalyClient::StorageSettings.allow_disk_access { source_repository.rugged } } + let(:new_oid) { new_commit_edit_old_file(source_rugged).oid } - it 'writes the ref' do - expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true) - expect(repository.commit(local_ref).sha).to eq(new_oid) - end + before do + source_rugged.branches.create(source_branch, new_oid) end - context 'when the commit exists locally' do - let(:source_branch) { 'master' } - let(:expected_oid) { SeedRepo::LastCommit::ID } - - it 'writes the ref' do - # Sanity check: the commit should already exist - expect(repository.commit(expected_oid)).not_to be_nil - - expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true) - expect(repository.commit(local_ref).sha).to eq(expected_oid) - end + it 'writes the ref' do + expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true) + expect(repository.commit(local_ref).sha).to eq(new_oid) end end - context 'when the branch does not exist' do - let(:source_branch) { 'definitely-not-master' } + context 'when the commit exists locally' do + let(:source_branch) { 'master' } + let(:expected_oid) { SeedRepo::LastCommit::ID } + + it 'writes the ref' do + # Sanity check: the commit should already exist + expect(repository.commit(expected_oid)).not_to be_nil - it 'does not write the ref' do - expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false) - expect(repository.commit(local_ref)).to be_nil + expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(true) + expect(repository.commit(local_ref).sha).to eq(expected_oid) end end end - it_behaves_like '#fetch_source_branch!' + context 'when the branch does not exist' do + let(:source_branch) { 'definitely-not-master' } - context 'without gitaly', :skip_gitaly_mock do - it_behaves_like '#fetch_source_branch!' + it 'does not write the ref' do + expect(repository.fetch_source_branch!(source_repository, source_branch, local_ref)).to eq(false) + expect(repository.commit(local_ref)).to be_nil + end end end diff --git a/spec/lib/gitlab/shell_spec.rb b/spec/lib/gitlab/shell_spec.rb index c435f988cdd..f8bf896950e 100644 --- a/spec/lib/gitlab/shell_spec.rb +++ b/spec/lib/gitlab/shell_spec.rb @@ -403,46 +403,36 @@ describe Gitlab::Shell do end describe '#create_repository' do - shared_examples '#create_repository' do - let(:repository_storage) { 'default' } - let(:repository_storage_path) do - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages[repository_storage].legacy_disk_path - end - end - let(:repo_name) { 'project/path' } - let(:created_path) { File.join(repository_storage_path, repo_name + '.git') } - - after do - FileUtils.rm_rf(created_path) + let(:repository_storage) { 'default' } + let(:repository_storage_path) do + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages[repository_storage].legacy_disk_path end + end + let(:repo_name) { 'project/path' } + let(:created_path) { File.join(repository_storage_path, repo_name + '.git') } - it 'creates a repository' do - expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy - - expect(File.stat(created_path).mode & 0o777).to eq(0o770) + after do + FileUtils.rm_rf(created_path) + end - hooks_path = File.join(created_path, 'hooks') - expect(File.lstat(hooks_path)).to be_symlink - expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path) - end + it 'creates a repository' do + expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_truthy - it 'returns false when the command fails' do - FileUtils.mkdir_p(File.dirname(created_path)) - # This file will block the creation of the repo's .git directory. That - # should cause #create_repository to fail. - FileUtils.touch(created_path) + expect(File.stat(created_path).mode & 0o777).to eq(0o770) - expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy - end + hooks_path = File.join(created_path, 'hooks') + expect(File.lstat(hooks_path)).to be_symlink + expect(File.realpath(hooks_path)).to eq(gitlab_shell_hooks_path) end - context 'with gitaly' do - it_behaves_like '#create_repository' - end + it 'returns false when the command fails' do + FileUtils.mkdir_p(File.dirname(created_path)) + # This file will block the creation of the repo's .git directory. That + # should cause #create_repository to fail. + FileUtils.touch(created_path) - context 'without gitaly', :skip_gitaly_mock do - it_behaves_like '#create_repository' + expect(gitlab_shell.create_repository(repository_storage, repo_name)).to be_falsy end end @@ -513,22 +503,12 @@ describe Gitlab::Shell do end end - shared_examples 'fetch_remote' do |gitaly_on| + describe '#fetch_remote' do def fetch_remote(ssh_auth = nil, prune = true) gitlab_shell.fetch_remote(repository.raw_repository, 'remote-name', ssh_auth: ssh_auth, prune: prune) end - def expect_gitlab_projects(fail = false, options = {}) - expect(gitlab_projects).to receive(:fetch_remote).with( - 'remote-name', - timeout, - options - ).and_return(!fail) - - allow(gitlab_projects).to receive(:output).and_return('error') if fail - end - - def expect_gitaly_call(fail, options = {}) + def expect_call(fail, options = {}) receive_fetch_remote = if fail receive(:fetch_remote).and_raise(GRPC::NotFound) @@ -539,16 +519,6 @@ describe Gitlab::Shell do expect_any_instance_of(Gitlab::GitalyClient::RepositoryService).to receive_fetch_remote end - if gitaly_on - def expect_call(fail, options = {}) - expect_gitaly_call(fail, options) - end - else - def expect_call(fail, options = {}) - expect_gitlab_projects(fail, options) - end - end - def build_ssh_auth(opts = {}) defaults = { ssh_import?: true, @@ -634,14 +604,6 @@ describe Gitlab::Shell do expect(fetch_remote(ssh_auth)).to be_truthy end end - end - - describe '#fetch_remote local', :skip_gitaly_mock do - it_should_behave_like 'fetch_remote', false - end - - describe '#fetch_remote gitaly' do - it_should_behave_like 'fetch_remote', true context 'gitaly call' do let(:remote_name) { 'remote-name' } @@ -683,25 +645,6 @@ describe Gitlab::Shell do end.to raise_error(Gitlab::Shell::Error, "error") end end - - context 'without gitaly', :disable_gitaly do - it 'returns true when the command succeeds' do - expect(gitlab_projects).to receive(:import_project).with(import_url, timeout) { true } - - result = gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url) - - expect(result).to be_truthy - end - - it 'raises an exception when the command fails' do - allow(gitlab_projects).to receive(:output) { 'error' } - expect(gitlab_projects).to receive(:import_project) { false } - - expect do - gitlab_shell.import_repository(project.repository_storage, project.disk_path, import_url) - end.to raise_error(Gitlab::Shell::Error, "error") - end - end end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 8fdcfe79fb5..b9e8c6c663a 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -36,22 +36,20 @@ describe Gitlab::Workhorse do allow(described_class).to receive(:git_archive_cache_disabled?).and_return(cache_disabled) end - context 'when Gitaly workhorse_archive feature is enabled' do - it 'sets the header correctly' do - key, command, params = decode_workhorse_header(subject) + it 'sets the header correctly' do + key, command, params = decode_workhorse_header(subject) - expect(key).to eq('Gitlab-Workhorse-Send-Data') - expect(command).to eq('git-archive') - expect(params).to include(gitaly_params) - end + expect(key).to eq('Gitlab-Workhorse-Send-Data') + expect(command).to eq('git-archive') + expect(params).to include(gitaly_params) + end - context 'when archive caching is disabled' do - let(:cache_disabled) { true } + context 'when archive caching is disabled' do + let(:cache_disabled) { true } - it 'tells workhorse not to use the cache' do - _, _, params = decode_workhorse_header(subject) - expect(params).to include({ 'DisableCache' => true }) - end + it 'tells workhorse not to use the cache' do + _, _, params = decode_workhorse_header(subject) + expect(params).to include({ 'DisableCache' => true }) end end -- cgit v1.2.1