From 88d2517e362accea4e51947e49a2587b5012dbc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Mon, 9 Oct 2017 18:25:24 -0300 Subject: Add `Gitlab::Git::Repository#fetch` command --- spec/lib/gitlab/git/repository_spec.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index bf6d199ebe2..8f1a7221f8f 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1685,6 +1685,21 @@ describe Gitlab::Git::Repository, seed_helper: true do end end + describe '#fetch' do + let(:git_path) { Gitlab.config.git.bin_path } + let(:remote_name) { 'my_remote' } + + subject { repository.fetch(remote_name) } + + it 'fetches the remote and returns true if the command was successful' do + expect(repository).to receive(:popen) + .with(%W(#{git_path} fetch #{remote_name}), repository.path) + .and_return(['', 0]) + + expect(subject).to be(true) + end + end + def create_remote_branch(repository, remote_name, branch_name, source_branch_name) source_branch = repository.branches.find { |branch| branch.name == source_branch_name } rugged = repository.rugged -- cgit v1.2.1 From 3f0233e5b531b44b2c276c8e8f536af6d2c15db3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Thu, 19 Oct 2017 13:27:03 -0300 Subject: Create a Wiki Repository's raw_repository properly --- app/models/project_wiki.rb | 2 +- app/models/repository.rb | 7 ++++--- spec/models/repository_spec.rb | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index bb7be29ef66..43de6809178 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -135,7 +135,7 @@ class ProjectWiki end def repository - @repository ||= Repository.new(full_path, @project, disk_path: disk_path) + @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true) end def default_branch diff --git a/app/models/repository.rb b/app/models/repository.rb index 44a1e9ce529..b1081db2e15 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -17,7 +17,7 @@ class Repository include Gitlab::ShellAdapter include RepositoryMirroring - attr_accessor :full_path, :disk_path, :project + attr_accessor :full_path, :disk_path, :project, :is_wiki delegate :ref_name_for_sha, to: :raw_repository @@ -72,11 +72,12 @@ class Repository end end - def initialize(full_path, project, disk_path: nil) + def initialize(full_path, project, disk_path: nil, is_wiki: false) @full_path = full_path @disk_path = disk_path || full_path @project = project @commit_cache = {} + @is_wiki = is_wiki end def ==(other) @@ -1141,7 +1142,7 @@ class Repository end def initialize_raw_repository - Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false)) + Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki)) end def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index d7c07676911..8a6aa767ce6 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -2298,4 +2298,24 @@ describe Repository do project.commit_by(oid: '1' * 40) end end + + describe '#raw_repository' do + subject { repository.raw_repository } + + it 'returns a Gitlab::Git::Repository representation of the repository' do + expect(subject).to be_a(Gitlab::Git::Repository) + expect(subject.relative_path).to eq(project.disk_path + '.git') + expect(subject.gl_repository).to eq("project-#{project.id}") + end + + context 'with a wiki repository' do + let(:repository) { project.wiki.repository } + + it 'creates a Gitlab::Git::Repository with the proper attributes' do + expect(subject).to be_a(Gitlab::Git::Repository) + expect(subject.relative_path).to eq(project.disk_path + '.wiki.git') + expect(subject.gl_repository).to eq("wiki-#{project.id}") + end + end + end end -- cgit v1.2.1 From dea6d054cdf3082651e356875e4a9225933a2383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Sat, 14 Oct 2017 22:52:07 -0300 Subject: Encapsulate git operations for mirroring in Gitlab::Git --- app/models/project.rb | 8 ++- app/models/repository.rb | 17 +----- app/services/projects/import_service.rb | 2 +- lib/github/import.rb | 4 +- lib/gitlab/git/repository.rb | 26 ++++++--- lib/gitlab/git/repository_mirroring.rb | 95 +++++++++++++++++++++++++++++++++ lib/gitlab/git/wiki.rb | 2 + spec/lib/gitlab/git/repository_spec.rb | 77 ++++++++++++++++++++------ 8 files changed, 188 insertions(+), 43 deletions(-) create mode 100644 lib/gitlab/git/repository_mirroring.rb diff --git a/app/models/project.rb b/app/models/project.rb index 413866b994a..27e71de298a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1679,6 +1679,10 @@ class Project < ActiveRecord::Base Gitlab::GlRepository.gl_repository(self, is_wiki) end + def reference_counter(wiki: false) + Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki)) + end + private def storage @@ -1697,11 +1701,11 @@ class Project < ActiveRecord::Base end def repo_reference_count - Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value + reference_counter.value end def wiki_reference_count - Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value + reference_counter(wiki: true).value end def check_repository_absence! diff --git a/app/models/repository.rb b/app/models/repository.rb index b1081db2e15..22912980695 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -966,21 +966,8 @@ class Repository run_git(args).first.lines.map(&:strip) end - def add_remote(name, url) - raw_repository.remote_add(name, url) - rescue Rugged::ConfigError - raw_repository.remote_update(name, url: url) - end - - def remove_remote(name) - raw_repository.remote_delete(name) - true - rescue Rugged::ConfigError - false - end - - def fetch_remote(remote, forced: false, no_tags: false) - gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags) + def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false) + gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) end def fetch_source_branch(source_repository, source_branch, local_ref) diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index c3bf0031409..455b302d819 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -44,7 +44,7 @@ module Projects else clone_repository end - rescue Gitlab::Shell::Error => e + rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e # Expire cache to prevent scenarios such as: # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true # 2. Retried import, repo is broken or not imported but +exists?+ still returns true diff --git a/lib/github/import.rb b/lib/github/import.rb index 76612799412..8cabbdec940 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -60,7 +60,9 @@ module Github project.repository.set_import_remote_as_mirror('github') project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') fetch_remote(forced: true) - rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + rescue Gitlab::Git::Repository::NoRepository, + Gitlab::Git::RepositoryMirroring::RemoteError, + Gitlab::Shell::Error => e error(:project, repo_url, e.message) raise Github::RepositoryFetchError end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4f9eac92d9a..182ffc96ef9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,6 +6,7 @@ require "rubygems/package" module Gitlab module Git class Repository + include Gitlab::Git::RepositoryMirroring include Gitlab::Git::Popen ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ @@ -898,16 +899,25 @@ module Gitlab end end - # Delete the specified remote from this repository. - def remote_delete(remote_name) - rugged.remotes.delete(remote_name) - nil + def add_remote(remote_name, url) + rugged.remotes.create(remote_name, url) + rescue Rugged::ConfigError + remote_update(remote_name, url: url) end - # Add a new remote to this repository. - def remote_add(remote_name, url) - rugged.remotes.create(remote_name, url) - nil + def remove_remote(remote_name) + # When a remote is deleted all its remote refs are deleted too, but in + # the case of mirrors we map its refs (that would usualy go under + # [remote_name]/) to the top level namespace. We clean the mapping so + # those don't get deleted. + if rugged.config["remote.#{remote_name}.mirror"] + rugged.config.delete("remote.#{remote_name}.fetch") + end + + rugged.remotes.delete(remote_name) + true + rescue Rugged::ConfigError + false end # Update the specified remote using the values in the +options+ hash diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb new file mode 100644 index 00000000000..637e7a0659c --- /dev/null +++ b/lib/gitlab/git/repository_mirroring.rb @@ -0,0 +1,95 @@ +module Gitlab + module Git + module RepositoryMirroring + IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze + IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze + MIRROR_REMOTE = 'mirror'.freeze + + RemoteError = Class.new(StandardError) + + def set_remote_as_mirror(remote_name) + # This is used to define repository as equivalent as "git clone --mirror" + rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' + rugged.config["remote.#{remote_name}.mirror"] = true + rugged.config["remote.#{remote_name}.prune"] = true + end + + def set_import_remote_as_mirror(remote_name) + # Add first fetch with Rugged so it does not create its own. + rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS + + add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) + + rugged.config["remote.#{remote_name}.mirror"] = true + rugged.config["remote.#{remote_name}.prune"] = true + end + + def add_remote_fetch_config(remote_name, refspec) + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + + def fetch_mirror(url) + add_remote(MIRROR_REMOTE, url) + set_remote_as_mirror(MIRROR_REMOTE) + fetch(MIRROR_REMOTE) + remove_remote(MIRROR_REMOTE) + end + + def remote_tags(remote) + # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n" + # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...] + list_remote_tags(remote).map do |line| + target, path = line.strip.split("\t") + + # When the remote repo does not have tags. + if target.nil? || path.nil? + Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" + return [] + end + + name = path.split('/', 3).last + # We're only interested in tag references + # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name + next if name =~ /\^\{\}\Z/ + + target_commit = Gitlab::Git::Commit.find(self, target) + Gitlab::Git::Tag.new(self, name, target, target_commit) + end.compact + end + + def remote_branches(remote_name) + branches = [] + + rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| + name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '') + + begin + target_commit = Gitlab::Git::Commit.find(self, ref.target) + branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end + + branches + end + + private + + def list_remote_tags(remote) + tag_list, exit_code, error = nil + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote}) + + Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| + tag_list = stdout.read + error = stderr.read + exit_code = wait_thr.value.exitstatus + end + + raise RemoteError, error unless exit_code.zero? + + tag_list.split('\n') + end + end + end +end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 45362ac438b..fe901d049d4 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -10,6 +10,8 @@ module Gitlab end PageBlob = Struct.new(:name) + attr_reader :repository + def self.default_ref 'master' end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 8f1a7221f8f..1d4d0c300eb 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -559,10 +559,10 @@ describe Gitlab::Git::Repository, seed_helper: true do end end - describe "#remote_delete" do + describe "#remove_remote" do before(:all) do @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') - @repo.remote_delete("expendable") + @repo.remove_remote("expendable") end it "should remove the remote" do @@ -575,14 +575,16 @@ describe Gitlab::Git::Repository, seed_helper: true do end end - describe "#remote_add" do + describe "#remote_update" do before(:all) do @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') - @repo.remote_add("new_remote", SeedHelper::GITLAB_GIT_TEST_REPO_URL) + @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH) end it "should add the remote" do - expect(@repo.rugged.remotes.each_name.to_a).to include("new_remote") + expect(@repo.rugged.remotes["expendable"].url).to( + eq(TEST_NORMAL_REPO_PATH) + ) end after(:all) do @@ -591,21 +593,58 @@ describe Gitlab::Git::Repository, seed_helper: true do end end - describe "#remote_update" do - before(:all) do - @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') - @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH) + describe '#fetch_mirror' do + let(:new_repository) do + Gitlab::Git::Repository.new('default', 'my_project.git', '') end - it "should add the remote" do - expect(@repo.rugged.remotes["expendable"].url).to( - eq(TEST_NORMAL_REPO_PATH) - ) + subject { new_repository.fetch_mirror(repository.path) } + + before do + Gitlab::Shell.new.add_repository('default', 'my_project') end - after(:all) do - FileUtils.rm_rf(TEST_MUTABLE_REPO_PATH) - ensure_seeds + after do + Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project') + end + + it 'fetches a url as a mirror remote' do + subject + + expect(refs(new_repository.path)).to eq(refs(repository.path)) + end + + context 'with keep-around refs' do + let(:sha) { SeedRepo::Commit::ID } + let(:keep_around_ref) { "refs/keep-around/#{sha}" } + let(:tmp_ref) { "refs/tmp/#{SecureRandom.hex}" } + + before do + repository.rugged.references.create(keep_around_ref, sha, force: true) + repository.rugged.references.create(tmp_ref, sha, force: true) + end + + it 'includes the temporary and keep-around refs' do + subject + + expect(refs(new_repository.path)).to include(keep_around_ref) + expect(refs(new_repository.path)).to include(tmp_ref) + end + end + end + + describe '#remote_tags' do + let(:target_commit_id) { SeedRepo::Commit::ID } + + subject { repository.remote_tags('upstream') } + + it 'gets the remote tags' do + expect(repository).to receive(:list_remote_tags).with('upstream') + .and_return(["#{target_commit_id}\trefs/tags/v0.0.1\n"]) + + expect(subject.first).to be_an_instance_of(Gitlab::Git::Tag) + expect(subject.first.name).to eq('v0.0.1') + expect(subject.first.dereferenced_target.id).to eq(target_commit_id) end end @@ -1775,4 +1814,10 @@ describe Gitlab::Git::Repository, seed_helper: true do sha = Rugged::Commit.create(repo, options) repo.lookup(sha) end + + def refs(dir) + IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line| + line.split("\t").last + end + end end -- cgit v1.2.1 From fe4874c43d3387aed52d95bc2892b27e551ac7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Sat, 14 Oct 2017 22:53:14 -0300 Subject: Use Gitlab::Git operations for repository mirroring --- app/models/concerns/repository_mirroring.rb | 32 ----------------------------- app/models/repository.rb | 1 - 2 files changed, 33 deletions(-) delete mode 100644 app/models/concerns/repository_mirroring.rb diff --git a/app/models/concerns/repository_mirroring.rb b/app/models/concerns/repository_mirroring.rb deleted file mode 100644 index f6aba91bc4c..00000000000 --- a/app/models/concerns/repository_mirroring.rb +++ /dev/null @@ -1,32 +0,0 @@ -module RepositoryMirroring - IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze - IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze - - def set_remote_as_mirror(name) - # This is used to define repository as equivalent as "git clone --mirror" - raw_repository.rugged.config["remote.#{name}.fetch"] = 'refs/*:refs/*' - raw_repository.rugged.config["remote.#{name}.mirror"] = true - raw_repository.rugged.config["remote.#{name}.prune"] = true - end - - def set_import_remote_as_mirror(remote_name) - # Add first fetch with Rugged so it does not create its own. - raw_repository.rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS - - add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) - - raw_repository.rugged.config["remote.#{remote_name}.mirror"] = true - raw_repository.rugged.config["remote.#{remote_name}.prune"] = true - end - - def add_remote_fetch_config(remote_name, refspec) - run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) - end - - def fetch_mirror(remote, url) - add_remote(remote, url) - set_remote_as_mirror(remote) - fetch_remote(remote, forced: true) - remove_remote(remote) - end -end diff --git a/app/models/repository.rb b/app/models/repository.rb index 22912980695..69cddb36b2e 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -15,7 +15,6 @@ class Repository ].freeze include Gitlab::ShellAdapter - include RepositoryMirroring attr_accessor :full_path, :disk_path, :project, :is_wiki -- cgit v1.2.1