diff options
author | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2016-07-27 15:28:03 +0000 |
---|---|---|
committer | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2016-07-27 15:28:03 +0000 |
commit | ead20c033a56be1e26a477bbd8802122f743fb24 (patch) | |
tree | d929996b6ebdc342bcb1c60b758a54b95725ee93 | |
parent | 87b388232505b1715a1d7448dc393a383cd7a53f (diff) | |
parent | 0625d469d6645d90eeab18449b70b6d2463085ce (diff) | |
download | gitlab-shell-ead20c033a56be1e26a477bbd8802122f743fb24.tar.gz |
Merge branch 'mv-storage' into 'master'
Add command to move repositories between repository storages
Necessary for https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/533
/cc @pcarranza
See merge request !71
-rw-r--r-- | CHANGELOG | 4 | ||||
-rwxr-xr-x | hooks/pre-receive | 8 | ||||
-rw-r--r-- | lib/gitlab_post_receive.rb | 3 | ||||
-rw-r--r-- | lib/gitlab_projects.rb | 52 | ||||
-rw-r--r-- | lib/gitlab_reference_counter.rb | 52 | ||||
-rw-r--r-- | spec/gitlab_post_receive_spec.rb | 33 | ||||
-rw-r--r-- | spec/gitlab_projects_spec.rb | 44 | ||||
-rw-r--r-- | spec/gitlab_reference_counter_spec.rb | 38 |
8 files changed, 232 insertions, 2 deletions
@@ -1,3 +1,7 @@ +v3.3.0 + - Track ongoing push commands + - Add command to move repositories between repository storages + v3.2.1 - Allow gitlab-project's fork-project command to fork projects between different repository storages diff --git a/hooks/pre-receive b/hooks/pre-receive index 66cb97d..09fa42b 100755 --- a/hooks/pre-receive +++ b/hooks/pre-receive @@ -9,10 +9,16 @@ protocol = ENV.delete('GL_PROTOCOL') repo_path = Dir.pwd require_relative '../lib/gitlab_custom_hook' +require_relative '../lib/gitlab_reference_counter' require_relative '../lib/gitlab_access' +# It's important that on pre-receive `increase_reference_counter` gets executed +# last so that it only runs if everything else succeeded. On post-receive on the +# other hand, we run GitlabPostReceive first because the push is already done +# and we don't want to skip it if the custom hook fails. if GitlabAccess.new(repo_path, key_id, refs, protocol).exec && - GitlabCustomHook.new.pre_receive(refs, repo_path) + GitlabCustomHook.new.pre_receive(refs, repo_path) && + GitlabReferenceCounter.new(repo_path).increase exit 0 else exit 1 diff --git a/lib/gitlab_post_receive.rb b/lib/gitlab_post_receive.rb index ebcf317..7874c85 100644 --- a/lib/gitlab_post_receive.rb +++ b/lib/gitlab_post_receive.rb @@ -1,5 +1,6 @@ require_relative 'gitlab_init' require_relative 'gitlab_net' +require_relative 'gitlab_reference_counter' require 'json' require 'base64' require 'securerandom' @@ -28,7 +29,7 @@ class GitlabPostReceive nil end - result + result && GitlabReferenceCounter.new(repo_path).decrease end protected diff --git a/lib/gitlab_projects.rb b/lib/gitlab_projects.rb index 4e6f271..a40cc4f 100644 --- a/lib/gitlab_projects.rb +++ b/lib/gitlab_projects.rb @@ -4,6 +4,7 @@ require 'open3' require_relative 'gitlab_config' require_relative 'gitlab_logger' +require_relative 'gitlab_reference_counter' class GitlabProjects GLOBAL_HOOKS_DIRECTORY = File.join(ROOT_PATH, 'hooks') @@ -55,6 +56,7 @@ class GitlabProjects when 'list-projects'; puts list_projects when 'rm-project'; rm_project when 'mv-project'; mv_project + when 'mv-storage'; mv_storage when 'import-project'; import_project when 'fork-project'; fork_project when 'fetch-remote'; fetch_remote @@ -285,6 +287,42 @@ class GitlabProjects FileUtils.mv(full_path, new_full_path) end + # Move repository from one storage path to another + # + # Wont work if target namespace directory does not exist in the new storage path + # + def mv_storage + new_storage = ARGV.shift + + unless new_storage + $logger.error "mv-storage failed: no destination storage path provided." + return false + end + + new_full_path = File.join(new_storage, project_name) + + # verify that the source repo exists + unless File.exists?(full_path) + $logger.error "mv-storage failed: source path <#{full_path}> does not exist." + return false + end + + # Make sure the destination directory exists + FileUtils.mkdir_p(new_full_path) + + # Make sure the source path ends with a slash so that rsync copies the + # contents of the directory, as opposed to copying the directory by name + source_path = File.join(full_path, '') + + if wait_for_pushes + $logger.info "Syncing project #{@project_name} from <#{full_path}> to <#{new_full_path}>." + system(*%W(rsync -a --delete #{source_path} #{new_full_path})) + else + $logger.error "mv-storage failed: source path <#{full_path}> is waiting for pushes to finish." + false + end + end + def fork_project destination_repos_path = ARGV.shift @@ -329,4 +367,18 @@ class GitlabProjects cmd = %W(git --git-dir=#{full_path} gc) system(*cmd) end + + def wait_for_pushes + # Try for 30 seconds, polling every 10 + 3.times do + return true if gitlab_reference_counter.value == 0 + sleep 10 + end + + false + end + + def gitlab_reference_counter + @gitlab_reference_counter ||= GitlabReferenceCounter.new(full_path) + end end diff --git a/lib/gitlab_reference_counter.rb b/lib/gitlab_reference_counter.rb new file mode 100644 index 0000000..dc08f42 --- /dev/null +++ b/lib/gitlab_reference_counter.rb @@ -0,0 +1,52 @@ +require_relative 'gitlab_init' +require_relative 'gitlab_net' + +class GitlabReferenceCounter + REFERENCE_EXPIRE_TIME = 600 + + attr_reader :path, :key + + def initialize(path) + @path = path + @key = "git-receive-pack-reference-counter:#{path}" + end + + def value + (redis_client.get(key) || 0).to_i + end + + def increase + redis_cmd do + redis_client.incr(key) + redis_client.expire(key, REFERENCE_EXPIRE_TIME) + end + end + + def decrease + redis_cmd do + current_value = redis_client.decr(key) + if current_value < 0 + $logger.warn "Reference counter for #{path} decreased when its value was less than 1. Reseting the counter." + redis_client.del(key) + end + end + end + + private + + def redis_client + @redis_client ||= GitlabNet.new.redis_client + end + + def redis_cmd + begin + yield + true + rescue => e + message = "GitLab: An unexpected error occurred in writing to Redis: #{e}" + $stderr.puts message + $logger.error message + false + end + end +end diff --git a/spec/gitlab_post_receive_spec.rb b/spec/gitlab_post_receive_spec.rb index 26b1037..6762d49 100644 --- a/spec/gitlab_post_receive_spec.rb +++ b/spec/gitlab_post_receive_spec.rb @@ -25,6 +25,11 @@ describe GitlabPostReceive do before do allow_any_instance_of(GitlabNet).to receive(:redis_client).and_return(redis_client) + allow_any_instance_of(GitlabReferenceCounter).to receive(:redis_client).and_return(redis_client) + allow(redis_client).to receive(:get).and_return(1) + allow(redis_client).to receive(:incr).and_return(true) + allow(redis_client).to receive(:decr).and_return(0) + allow(redis_client).to receive(:rpush).and_return(true) end it "prints the broadcast message" do @@ -59,6 +64,34 @@ describe GitlabPostReceive do gitlab_post_receive.exec end + context 'reference counter' do + it 'decreases the reference counter for the project' do + expect_any_instance_of(GitlabReferenceCounter).to receive(:decrease).and_return(true) + + gitlab_post_receive.exec + end + + context "when the redis command succeeds" do + before do + allow(redis_client).to receive(:decr).and_return(0) + end + + it "returns true" do + expect(gitlab_post_receive.exec).to eq(true) + end + end + + context "when the redis command fails" do + before do + allow(redis_client).to receive(:decr).and_raise('Fail') + end + + it "returns false" do + expect(gitlab_post_receive.exec).to eq(false) + end + end + end + context "when the redis command succeeds" do before do diff --git a/spec/gitlab_projects_spec.rb b/spec/gitlab_projects_spec.rb index a06b4d6..19646e3 100644 --- a/spec/gitlab_projects_spec.rb +++ b/spec/gitlab_projects_spec.rb @@ -205,6 +205,50 @@ describe GitlabProjects do end end + describe :mv_storage do + let(:alternative_storage_path) { File.join(ROOT_PATH, 'tmp', 'alternative') } + let(:gl_projects) { build_gitlab_projects('mv-storage', tmp_repos_path, repo_name, alternative_storage_path) } + let(:new_repo_path) { File.join(alternative_storage_path, repo_name) } + + before do + FileUtils.mkdir_p(tmp_repo_path) + FileUtils.mkdir_p(alternative_storage_path) + allow_any_instance_of(GitlabReferenceCounter).to receive(:value).and_return(0) + end + + after { FileUtils.rm_rf(alternative_storage_path) } + + it "should rsync a repo directory" do + File.exists?(tmp_repo_path).should be_true + gl_projects.exec + File.exists?(new_repo_path).should be_true + end + + it "should fail if no destination path is provided" do + incomplete = build_gitlab_projects('mv-storage', tmp_repos_path, repo_name) + $logger.should_receive(:error).with("mv-storage failed: no destination storage path provided.") + incomplete.exec.should be_false + end + + it "should fail if the source path doesn't exist" do + bad_source = build_gitlab_projects('mv-storage', tmp_repos_path, 'bad-src.git', alternative_storage_path) + $logger.should_receive(:error).with("mv-storage failed: source path <#{tmp_repos_path}/bad-src.git> does not exist.") + bad_source.exec.should be_false + end + + it "should fail if there are pushes ongoing" do + allow_any_instance_of(GitlabReferenceCounter).to receive(:value).and_return(1) + $logger.should_receive(:error).with("mv-storage failed: source path <#{tmp_repo_path}> is waiting for pushes to finish.") + gl_projects.exec.should be_false + end + + it "should log an mv-storage event" do + message = "Syncing project #{repo_name} from <#{tmp_repo_path}> to <#{new_repo_path}>." + $logger.should_receive(:info).with(message) + gl_projects.exec + end + end + describe :import_project do context 'success import' do let(:gl_projects) { build_gitlab_projects('import-project', tmp_repos_path, repo_name, 'https://github.com/randx/six.git') } diff --git a/spec/gitlab_reference_counter_spec.rb b/spec/gitlab_reference_counter_spec.rb new file mode 100644 index 0000000..5be53ff --- /dev/null +++ b/spec/gitlab_reference_counter_spec.rb @@ -0,0 +1,38 @@ +# coding: utf-8 +require 'spec_helper' +require 'gitlab_reference_counter' + +describe GitlabReferenceCounter do + let(:redis_client) { double('redis_client') } + let(:reference_counter_key) { "git-receive-pack-reference-counter:/test/path" } + let(:gitlab_reference_counter) { GitlabReferenceCounter.new('/test/path') } + + before do + allow(gitlab_reference_counter).to receive(:redis_client).and_return(redis_client) + $logger = double('logger').as_null_object + end + + it 'increases and set the expire time of a reference count for a path' do + expect(redis_client).to receive(:incr).with(reference_counter_key) + expect(redis_client).to receive(:expire).with(reference_counter_key, GitlabReferenceCounter::REFERENCE_EXPIRE_TIME) + expect(gitlab_reference_counter.increase).to be(true) + end + + it 'decreases the reference count for a path' do + allow(redis_client).to receive(:decr).and_return(0) + expect(redis_client).to receive(:decr).with(reference_counter_key) + expect(gitlab_reference_counter.decrease).to be(true) + end + + it 'warns if attempting to decrease a counter with a value of one or less, and resets the counter' do + expect(redis_client).to receive(:decr).and_return(-1) + expect(redis_client).to receive(:del) + expect($logger).to receive(:warn).with("Reference counter for /test/path decreased when its value was less than 1. Reseting the counter.") + expect(gitlab_reference_counter.decrease).to be(true) + end + + it 'get the reference count for a path' do + allow(redis_client).to receive(:get).and_return(1) + expect(gitlab_reference_counter.value).to be(1) + end +end |