summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlejandro Rodríguez <alejorro70@gmail.com>2017-10-14 22:52:07 -0300
committerAlejandro Rodríguez <alejorro70@gmail.com>2017-11-03 14:33:24 -0300
commitdea6d054cdf3082651e356875e4a9225933a2383 (patch)
tree3bdbdd89b6d426ed9974cdaf997cacbef6a3e8c5
parent3f0233e5b531b44b2c276c8e8f536af6d2c15db3 (diff)
downloadgitlab-ce-dea6d054cdf3082651e356875e4a9225933a2383.tar.gz
Encapsulate git operations for mirroring in Gitlab::Git
-rw-r--r--app/models/project.rb8
-rw-r--r--app/models/repository.rb17
-rw-r--r--app/services/projects/import_service.rb2
-rw-r--r--lib/github/import.rb4
-rw-r--r--lib/gitlab/git/repository.rb26
-rw-r--r--lib/gitlab/git/repository_mirroring.rb95
-rw-r--r--lib/gitlab/git/wiki.rb2
-rw-r--r--spec/lib/gitlab/git/repository_spec.rb77
8 files changed, 188 insertions, 43 deletions
diff --git a/app/models/project.rb b/app/models/project.rb
index 413866b994a..27e71de298a 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1679,6 +1679,10 @@ class Project < ActiveRecord::Base
Gitlab::GlRepository.gl_repository(self, is_wiki)
end
+ def reference_counter(wiki: false)
+ Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
+ end
+
private
def storage
@@ -1697,11 +1701,11 @@ class Project < ActiveRecord::Base
end
def repo_reference_count
- Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
+ reference_counter.value
end
def wiki_reference_count
- Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
+ reference_counter(wiki: true).value
end
def check_repository_absence!
diff --git a/app/models/repository.rb b/app/models/repository.rb
index b1081db2e15..22912980695 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -966,21 +966,8 @@ class Repository
run_git(args).first.lines.map(&:strip)
end
- def add_remote(name, url)
- raw_repository.remote_add(name, url)
- rescue Rugged::ConfigError
- raw_repository.remote_update(name, url: url)
- end
-
- def remove_remote(name)
- raw_repository.remote_delete(name)
- true
- rescue Rugged::ConfigError
- false
- end
-
- def fetch_remote(remote, forced: false, no_tags: false)
- gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags)
+ def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
+ gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
end
def fetch_source_branch(source_repository, source_branch, local_ref)
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index c3bf0031409..455b302d819 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -44,7 +44,7 @@ module Projects
else
clone_repository
end
- rescue Gitlab::Shell::Error => e
+ rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
# Expire cache to prevent scenarios such as:
# 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
# 2. Retried import, repo is broken or not imported but +exists?+ still returns true
diff --git a/lib/github/import.rb b/lib/github/import.rb
index 76612799412..8cabbdec940 100644
--- a/lib/github/import.rb
+++ b/lib/github/import.rb
@@ -60,7 +60,9 @@ module Github
project.repository.set_import_remote_as_mirror('github')
project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head')
fetch_remote(forced: true)
- rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
+ rescue Gitlab::Git::Repository::NoRepository,
+ Gitlab::Git::RepositoryMirroring::RemoteError,
+ Gitlab::Shell::Error => e
error(:project, repo_url, e.message)
raise Github::RepositoryFetchError
end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 4f9eac92d9a..182ffc96ef9 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -6,6 +6,7 @@ require "rubygems/package"
module Gitlab
module Git
class Repository
+ include Gitlab::Git::RepositoryMirroring
include Gitlab::Git::Popen
ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
@@ -898,16 +899,25 @@ module Gitlab
end
end
- # Delete the specified remote from this repository.
- def remote_delete(remote_name)
- rugged.remotes.delete(remote_name)
- nil
+ def add_remote(remote_name, url)
+ rugged.remotes.create(remote_name, url)
+ rescue Rugged::ConfigError
+ remote_update(remote_name, url: url)
end
- # Add a new remote to this repository.
- def remote_add(remote_name, url)
- rugged.remotes.create(remote_name, url)
- nil
+ def remove_remote(remote_name)
+ # When a remote is deleted all its remote refs are deleted too, but in
+ # the case of mirrors we map its refs (that would usualy go under
+ # [remote_name]/) to the top level namespace. We clean the mapping so
+ # those don't get deleted.
+ if rugged.config["remote.#{remote_name}.mirror"]
+ rugged.config.delete("remote.#{remote_name}.fetch")
+ end
+
+ rugged.remotes.delete(remote_name)
+ true
+ rescue Rugged::ConfigError
+ false
end
# Update the specified remote using the values in the +options+ hash
diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb
new file mode 100644
index 00000000000..637e7a0659c
--- /dev/null
+++ b/lib/gitlab/git/repository_mirroring.rb
@@ -0,0 +1,95 @@
+module Gitlab
+ module Git
+ module RepositoryMirroring
+ IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
+ IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
+ MIRROR_REMOTE = 'mirror'.freeze
+
+ RemoteError = Class.new(StandardError)
+
+ def set_remote_as_mirror(remote_name)
+ # This is used to define repository as equivalent as "git clone --mirror"
+ rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
+ rugged.config["remote.#{remote_name}.mirror"] = true
+ rugged.config["remote.#{remote_name}.prune"] = true
+ end
+
+ def set_import_remote_as_mirror(remote_name)
+ # Add first fetch with Rugged so it does not create its own.
+ rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
+
+ add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
+
+ rugged.config["remote.#{remote_name}.mirror"] = true
+ rugged.config["remote.#{remote_name}.prune"] = true
+ end
+
+ def add_remote_fetch_config(remote_name, refspec)
+ run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+ end
+
+ def fetch_mirror(url)
+ add_remote(MIRROR_REMOTE, url)
+ set_remote_as_mirror(MIRROR_REMOTE)
+ fetch(MIRROR_REMOTE)
+ remove_remote(MIRROR_REMOTE)
+ end
+
+ def remote_tags(remote)
+ # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
+ # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
+ list_remote_tags(remote).map do |line|
+ target, path = line.strip.split("\t")
+
+ # When the remote repo does not have tags.
+ if target.nil? || path.nil?
+ Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}"
+ return []
+ end
+
+ name = path.split('/', 3).last
+ # We're only interested in tag references
+ # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name
+ next if name =~ /\^\{\}\Z/
+
+ target_commit = Gitlab::Git::Commit.find(self, target)
+ Gitlab::Git::Tag.new(self, name, target, target_commit)
+ end.compact
+ end
+
+ def remote_branches(remote_name)
+ branches = []
+
+ rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
+ name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
+
+ begin
+ target_commit = Gitlab::Git::Commit.find(self, ref.target)
+ branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit)
+ rescue Rugged::ReferenceError
+ # Omit invalid branch
+ end
+ end
+
+ branches
+ end
+
+ private
+
+ def list_remote_tags(remote)
+ tag_list, exit_code, error = nil
+ cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote})
+
+ Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
+ tag_list = stdout.read
+ error = stderr.read
+ exit_code = wait_thr.value.exitstatus
+ end
+
+ raise RemoteError, error unless exit_code.zero?
+
+ tag_list.split('\n')
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index 45362ac438b..fe901d049d4 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -10,6 +10,8 @@ module Gitlab
end
PageBlob = Struct.new(:name)
+ attr_reader :repository
+
def self.default_ref
'master'
end
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index 8f1a7221f8f..1d4d0c300eb 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -559,10 +559,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
- describe "#remote_delete" do
+ describe "#remove_remote" do
before(:all) do
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
- @repo.remote_delete("expendable")
+ @repo.remove_remote("expendable")
end
it "should remove the remote" do
@@ -575,14 +575,16 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
- describe "#remote_add" do
+ describe "#remote_update" do
before(:all) do
@repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
- @repo.remote_add("new_remote", SeedHelper::GITLAB_GIT_TEST_REPO_URL)
+ @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH)
end
it "should add the remote" do
- expect(@repo.rugged.remotes.each_name.to_a).to include("new_remote")
+ expect(@repo.rugged.remotes["expendable"].url).to(
+ eq(TEST_NORMAL_REPO_PATH)
+ )
end
after(:all) do
@@ -591,21 +593,58 @@ describe Gitlab::Git::Repository, seed_helper: true do
end
end
- describe "#remote_update" do
- before(:all) do
- @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
- @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH)
+ describe '#fetch_mirror' do
+ let(:new_repository) do
+ Gitlab::Git::Repository.new('default', 'my_project.git', '')
end
- it "should add the remote" do
- expect(@repo.rugged.remotes["expendable"].url).to(
- eq(TEST_NORMAL_REPO_PATH)
- )
+ subject { new_repository.fetch_mirror(repository.path) }
+
+ before do
+ Gitlab::Shell.new.add_repository('default', 'my_project')
end
- after(:all) do
- FileUtils.rm_rf(TEST_MUTABLE_REPO_PATH)
- ensure_seeds
+ after do
+ Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project')
+ end
+
+ it 'fetches a url as a mirror remote' do
+ subject
+
+ expect(refs(new_repository.path)).to eq(refs(repository.path))
+ end
+
+ context 'with keep-around refs' do
+ let(:sha) { SeedRepo::Commit::ID }
+ let(:keep_around_ref) { "refs/keep-around/#{sha}" }
+ let(:tmp_ref) { "refs/tmp/#{SecureRandom.hex}" }
+
+ before do
+ repository.rugged.references.create(keep_around_ref, sha, force: true)
+ repository.rugged.references.create(tmp_ref, sha, force: true)
+ end
+
+ it 'includes the temporary and keep-around refs' do
+ subject
+
+ expect(refs(new_repository.path)).to include(keep_around_ref)
+ expect(refs(new_repository.path)).to include(tmp_ref)
+ end
+ end
+ end
+
+ describe '#remote_tags' do
+ let(:target_commit_id) { SeedRepo::Commit::ID }
+
+ subject { repository.remote_tags('upstream') }
+
+ it 'gets the remote tags' do
+ expect(repository).to receive(:list_remote_tags).with('upstream')
+ .and_return(["#{target_commit_id}\trefs/tags/v0.0.1\n"])
+
+ expect(subject.first).to be_an_instance_of(Gitlab::Git::Tag)
+ expect(subject.first.name).to eq('v0.0.1')
+ expect(subject.first.dereferenced_target.id).to eq(target_commit_id)
end
end
@@ -1775,4 +1814,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
+
+ def refs(dir)
+ IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
+ line.split("\t").last
+ end
+ end
end