diff options
author | Rémy Coutable <remy@rymai.me> | 2019-02-04 14:11:04 +0000 |
---|---|---|
committer | Rémy Coutable <remy@rymai.me> | 2019-02-04 14:11:04 +0000 |
commit | 5c583c8e87f6b299b3a7ea07dded0972740092e9 (patch) | |
tree | 63fda670aa22f49d062e23e10dc13f9983e2d5f1 /spec/tasks | |
parent | cc41a771832a9d44c9a87e25aa784cb904d03fd5 (diff) | |
parent | 6b0b14f81d6def6d74b303cd27fef6f98aaabfd0 (diff) | |
download | gitlab-ce-5c583c8e87f6b299b3a7ea07dded0972740092e9.tar.gz |
Merge branch 'master' into '46448-add-timestamps-for-each-stage-of-gitlab-rake-gitlab-backup-restore'
# Conflicts:
# spec/tasks/gitlab/backup_rake_spec.rb
Diffstat (limited to 'spec/tasks')
-rw-r--r-- | spec/tasks/cache/clear/redis_spec.rb | 5 | ||||
-rw-r--r-- | spec/tasks/gitlab/backup_rake_spec.rb | 42 | ||||
-rw-r--r-- | spec/tasks/gitlab/check_rake_spec.rb | 108 | ||||
-rw-r--r-- | spec/tasks/gitlab/cleanup_rake_spec.rb | 121 | ||||
-rw-r--r-- | spec/tasks/gitlab/db_rake_spec.rb | 43 | ||||
-rw-r--r-- | spec/tasks/gitlab/git_rake_spec.rb | 27 | ||||
-rw-r--r-- | spec/tasks/gitlab/gitaly_rake_spec.rb | 65 | ||||
-rw-r--r-- | spec/tasks/gitlab/shell_rake_spec.rb | 8 | ||||
-rw-r--r-- | spec/tasks/gitlab/storage_rake_spec.rb | 65 | ||||
-rw-r--r-- | spec/tasks/gitlab/traces_rake_spec.rb | 112 | ||||
-rw-r--r-- | spec/tasks/gitlab/uploads/migrate_rake_spec.rb | 8 | ||||
-rw-r--r-- | spec/tasks/gitlab/web_hook_rake_spec.rb | 92 |
12 files changed, 519 insertions, 177 deletions
diff --git a/spec/tasks/cache/clear/redis_spec.rb b/spec/tasks/cache/clear/redis_spec.rb index cca2b864e9b..97c8c943f3a 100644 --- a/spec/tasks/cache/clear/redis_spec.rb +++ b/spec/tasks/cache/clear/redis_spec.rb @@ -6,7 +6,10 @@ describe 'clearing redis cache' do end describe 'clearing pipeline status cache' do - let(:pipeline_status) { create(:ci_pipeline).project.pipeline_status } + let(:pipeline_status) do + project = create(:project, :repository) + create(:ci_pipeline, project: project).project.pipeline_status + end before do allow(pipeline_status).to receive(:loaded).and_return(nil) diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 70f3d77dc98..a8fae4a88a3 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -96,6 +96,27 @@ describe 'gitlab:app namespace rake task' do end end end + + context 'when the restore directory is not empty' do + before do + # We only need a backup of the repositories for this test + stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry') + end + + it 'removes stale data' do + expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout + + excluded_project = create(:project, :repository, name: 'mepmep') + + expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout + + raw_repo = excluded_project.repository.raw + + # The restore will not find the repository in the backup, but will create + # an empty one in its place + expect(raw_repo.empty?).to be(true) + end + end end # backup_restore task describe 'backup' do @@ -110,7 +131,9 @@ describe 'gitlab:app namespace rake task' do before do stub_env('SKIP', 'db') - path = File.join(project.repository.path_to_repo, 'custom_hooks') + path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(project.repository.path_to_repo, 'custom_hooks') + end FileUtils.mkdir_p(path) FileUtils.touch(File.join(path, "dummy.txt")) end @@ -131,7 +154,10 @@ describe 'gitlab:app namespace rake task' do expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout - expect(Dir.entries(File.join(project.repository.path, 'custom_hooks'))).to include("dummy.txt") + repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + project.repository.path + end + expect(Dir.entries(File.join(repo_path, 'custom_hooks'))).to include("dummy.txt") end end @@ -234,7 +260,7 @@ describe 'gitlab:app namespace rake task' do allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) - # Avoid asking gitaly about the root ref (which will fail beacuse of the + # Avoid asking gitaly about the root ref (which will fail because of the # mocked storages) allow_any_instance_of(Repository).to receive(:empty?).and_return(false) end @@ -252,10 +278,12 @@ describe 'gitlab:app namespace rake task' do FileUtils.mkdir_p(b_storage_dir) # Even when overriding the storage, we have to move it there, so it exists - FileUtils.mv( - File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'), - Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git') - ) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + FileUtils.mv( + File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'), + Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git') + ) + end expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout diff --git a/spec/tasks/gitlab/check_rake_spec.rb b/spec/tasks/gitlab/check_rake_spec.rb index 4eda618b6d6..06525e3c771 100644 --- a/spec/tasks/gitlab/check_rake_spec.rb +++ b/spec/tasks/gitlab/check_rake_spec.rb @@ -1,51 +1,101 @@ require 'rake_helper' -describe 'gitlab:ldap:check rake task' do - include LdapHelpers - +describe 'check.rake' do before do Rake.application.rake_require 'tasks/gitlab/check' stub_warn_user_is_not_gitlab end - context 'when LDAP is not enabled' do - it 'does not attempt to bind or search for users' do - expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers) - expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open) - - run_rake_task('gitlab:ldap:check') + shared_examples_for 'system check rake task' do + it 'runs the check' do + expect do + subject + end.to output(/Checking #{name} ... Finished/).to_stdout end end - context 'when LDAP is enabled' do - let(:ldap) { double(:ldap) } - let(:adapter) { ldap_adapter('ldapmain', ldap) } + describe 'gitlab:check rake task' do + subject { run_rake_task('gitlab:check') } + let(:name) { 'GitLab subtasks' } - before do - allow(Gitlab::Auth::LDAP::Config) - .to receive_messages( - enabled?: true, - providers: ['ldapmain'] - ) - allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter) - allow(adapter).to receive(:users).and_return([]) - end + it_behaves_like 'system check rake task' + end + + describe 'gitlab:gitlab_shell:check rake task' do + subject { run_rake_task('gitlab:gitlab_shell:check') } + let(:name) { 'GitLab Shell' } + + it_behaves_like 'system check rake task' + end + + describe 'gitlab:gitaly:check rake task' do + subject { run_rake_task('gitlab:gitaly:check') } + let(:name) { 'Gitaly' } + + it_behaves_like 'system check rake task' + end + + describe 'gitlab:sidekiq:check rake task' do + subject { run_rake_task('gitlab:sidekiq:check') } + let(:name) { 'Sidekiq' } - it 'attempts to bind using credentials' do - stub_ldap_config(has_auth?: true) + it_behaves_like 'system check rake task' + end - expect(ldap).to receive(:bind) + describe 'gitlab:incoming_email:check rake task' do + subject { run_rake_task('gitlab:incoming_email:check') } + let(:name) { 'Incoming Email' } - run_rake_task('gitlab:ldap:check') + it_behaves_like 'system check rake task' + end + + describe 'gitlab:ldap:check rake task' do + include LdapHelpers + + subject { run_rake_task('gitlab:ldap:check') } + let(:name) { 'LDAP' } + + it_behaves_like 'system check rake task' + + context 'when LDAP is not enabled' do + it 'does not attempt to bind or search for users' do + expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers) + expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open) + + subject + end end - it 'searches for 100 LDAP users' do - stub_ldap_config(uid: 'uid') + context 'when LDAP is enabled' do + let(:ldap) { double(:ldap) } + let(:adapter) { ldap_adapter('ldapmain', ldap) } + + before do + allow(Gitlab::Auth::LDAP::Config) + .to receive_messages( + enabled?: true, + providers: ['ldapmain'] + ) + allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter) + allow(adapter).to receive(:users).and_return([]) + end + + it 'attempts to bind using credentials' do + stub_ldap_config(has_auth?: true) + + expect(ldap).to receive(:bind) + + subject + end + + it 'searches for 100 LDAP users' do + stub_ldap_config(uid: 'uid') - expect(adapter).to receive(:users).with('uid', '*', 100) + expect(adapter).to receive(:users).with('uid', '*', 100) - run_rake_task('gitlab:ldap:check') + subject + end end end end diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb index 2bf873c923f..19794227d9f 100644 --- a/spec/tasks/gitlab/cleanup_rake_spec.rb +++ b/spec/tasks/gitlab/cleanup_rake_spec.rb @@ -5,7 +5,9 @@ describe 'gitlab:cleanup rake tasks' do Rake.application.rake_require 'tasks/gitlab/cleanup' end - describe 'cleanup' do + describe 'cleanup namespaces and repos' do + let(:gitlab_shell) { Gitlab::Shell.new } + let(:storage) { storages.keys.first } let(:storages) do { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) @@ -17,53 +19,140 @@ describe 'gitlab:cleanup rake tasks' do end before do - FileUtils.mkdir(Settings.absolute('tmp/tests/default_storage')) allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) end after do - FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage')) + Gitlab::GitalyClient::StorageService.new(storage).delete_all_repositories end describe 'cleanup:repos' do before do - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/broken/project.git')) - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git')) + gitlab_shell.add_namespace(storage, 'broken/project.git') + gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git') end it 'moves it to an orphaned path' do - run_rake_task('gitlab:cleanup:repos') - repo_list = Dir['tmp/tests/default_storage/broken/*'] + now = Time.now + + Timecop.freeze(now) do + run_rake_task('gitlab:cleanup:repos') + repo_list = Gitlab::GitalyClient::StorageService.new(storage).list_directories(depth: 0) - expect(repo_list.first).to include('+orphaned+') + expect(repo_list.last).to include("broken+orphaned+#{now.to_i}") + end end it 'ignores @hashed repos' do run_rake_task('gitlab:cleanup:repos') - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy + expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true) end end describe 'cleanup:dirs' do it 'removes missing namespaces' do - FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_1/project.git")) - FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_2/project.git")) - allow(Namespace).to receive(:pluck).and_return('namespace_1') + gitlab_shell.add_namespace(storage, "namespace_1/project.git") + gitlab_shell.add_namespace(storage, "namespace_2/project.git") + allow(Namespace).to receive(:pluck).and_return(['namespace_1']) stub_env('REMOVE', 'true') run_rake_task('gitlab:cleanup:dirs') - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_1'))).to be_truthy - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_2'))).to be_falsey + expect(gitlab_shell.exists?(storage, 'namespace_1')).to be(true) + expect(gitlab_shell.exists?(storage, 'namespace_2')).to be(false) end it 'ignores @hashed directory' do - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git')) + gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git') run_rake_task('gitlab:cleanup:dirs') - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy + expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true) + end + end + end + + # A single integration test that is redundant with one part of the + # Gitlab::Cleanup::ProjectUploads spec. + # + # Additionally, this tests DRY_RUN env var values, and the extra line of + # output that says you can disable DRY_RUN if it's enabled. + describe 'cleanup:project_uploads' do + let!(:logger) { double(:logger) } + + before do + expect(main_object).to receive(:logger).and_return(logger).at_least(1).times + + allow(logger).to receive(:info).at_least(1).times + allow(logger).to receive(:debug).at_least(1).times + end + + context 'with a fixable orphaned project upload file' do + let(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) } + let(:new_path) { orphaned.absolute_path } + let(:path) { File.join(FileUploader.root, 'some', 'wrong', 'location', orphaned.path) } + + before do + FileUtils.mkdir_p(File.dirname(path)) + FileUtils.mv(new_path, path) + end + + context 'with DRY_RUN disabled' do + before do + stub_env('DRY_RUN', 'false') + end + + it 'moves the file to its proper location' do + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(path)).to be_falsey + expect(File.exist?(new_path)).to be_truthy + end + + it 'logs action as done' do + expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up...") + expect(logger).to receive(:info).with("Did fix #{path} -> #{new_path}") + + run_rake_task('gitlab:cleanup:project_uploads') + end + end + + shared_examples_for 'does not move the file' do + it 'does not move the file' do + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(path)).to be_truthy + expect(File.exist?(new_path)).to be_falsey + end + + it 'logs action as able to be done' do + expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up. Dry run...") + expect(logger).to receive(:info).with("Can fix #{path} -> #{new_path}") + expect(logger).to receive(:info).with(/To clean up these files run this command with DRY_RUN=false/) + + run_rake_task('gitlab:cleanup:project_uploads') + end + end + + context 'with DRY_RUN explicitly enabled' do + before do + stub_env('DRY_RUN', 'true') + end + + it_behaves_like 'does not move the file' + end + + context 'with DRY_RUN set to an unknown value' do + before do + stub_env('DRY_RUN', 'foo') + end + + it_behaves_like 'does not move the file' + end + + context 'with DRY_RUN unset' do + it_behaves_like 'does not move the file' end end end diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index fc52c04e78d..5818892d56a 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -20,7 +20,7 @@ describe 'gitlab:db namespace rake task' do describe 'configure' do it 'invokes db:migrate when schema has already been loaded' do - allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['default']) + allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[table1 table2]) expect(Rake::Task['db:migrate']).to receive(:invoke) expect(Rake::Task['db:schema:load']).not_to receive(:invoke) expect(Rake::Task['db:seed_fu']).not_to receive(:invoke) @@ -35,6 +35,14 @@ describe 'gitlab:db namespace rake task' do expect { run_rake_task('gitlab:db:configure') }.not_to raise_error end + it 'invokes db:shema:load and db:seed_fu when there is only a single table present' do + allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['default']) + expect(Rake::Task['db:schema:load']).to receive(:invoke) + expect(Rake::Task['db:seed_fu']).to receive(:invoke) + expect(Rake::Task['db:migrate']).not_to receive(:invoke) + expect { run_rake_task('gitlab:db:configure') }.not_to raise_error + end + it 'does not invoke any other rake tasks during an error' do allow(ActiveRecord::Base).to receive(:connection).and_raise(RuntimeError, 'error') expect(Rake::Task['db:migrate']).not_to receive(:invoke) @@ -53,6 +61,39 @@ describe 'gitlab:db namespace rake task' do expect(Rake::Task['db:migrate']).not_to receive(:invoke) expect { run_rake_task('gitlab:db:configure') }.to raise_error(RuntimeError, 'error') end + + context 'SKIP_POST_DEPLOYMENT_MIGRATIONS environment variable set' do + let(:rails_paths) { { 'db' => ['db'], 'db/migrate' => ['db/migrate'] } } + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('SKIP_POST_DEPLOYMENT_MIGRATIONS').and_return true + + # Our environment has already been loaded, so we need to pretend like post_migrations were not + allow(Rails.application.config).to receive(:paths).and_return(rails_paths) + allow(ActiveRecord::Migrator).to receive(:migrations_paths).and_return(rails_paths['db/migrate'].dup) + end + + it 'adds post deployment migrations before schema load if the schema is not already loaded' do + allow(ActiveRecord::Base.connection).to receive(:tables).and_return([]) + expect(Gitlab::Database).to receive(:add_post_migrate_path_to_rails).and_call_original + expect(Rake::Task['db:schema:load']).to receive(:invoke) + expect(Rake::Task['db:seed_fu']).to receive(:invoke) + expect(Rake::Task['db:migrate']).not_to receive(:invoke) + expect { run_rake_task('gitlab:db:configure') }.not_to raise_error + expect(rails_paths['db/migrate'].include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(true) + end + + it 'ignores post deployment migrations when schema has already been loaded' do + allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[table1 table2]) + expect(Rake::Task['db:migrate']).to receive(:invoke) + expect(Gitlab::Database).not_to receive(:add_post_migrate_path_to_rails) + expect(Rake::Task['db:schema:load']).not_to receive(:invoke) + expect(Rake::Task['db:seed_fu']).not_to receive(:invoke) + expect { run_rake_task('gitlab:db:configure') }.not_to raise_error + expect(rails_paths['db/migrate'].include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(false) + end + end end def run_rake_task(task_name) diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb index 1efaecc63a5..57b006e1a39 100644 --- a/spec/tasks/gitlab/git_rake_spec.rb +++ b/spec/tasks/gitlab/git_rake_spec.rb @@ -1,41 +1,20 @@ require 'rake_helper' describe 'gitlab:git rake tasks' do - before(:all) do - @default_storage_hash = Gitlab.config.repositories.storages.default.to_h - end + let(:base_path) { 'tmp/tests/default_storage' } + let!(:project) { create(:project, :repository) } before do Rake.application.rake_require 'tasks/gitlab/git' - storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) } - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git')) - allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) allow_any_instance_of(String).to receive(:color) { |string, _color| string } stub_warn_user_is_not_gitlab end - after do - FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage')) - end - describe 'fsck' do it 'outputs the integrity check for a repo' do - expect { run_rake_task('gitlab:git:fsck') }.to output(%r{Performed Checking integrity at .*@hashed/1/2/test.git}).to_stdout - end - - it 'errors out about config.lock issues' do - FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/config.lock')) - - expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout - end - - it 'errors out about ref lock issues' do - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads')) - FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads/blah.lock')) - - expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout + expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed integrity check for/).to_stdout end end end diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb index 1e507c0236e..e6e4d9504d9 100644 --- a/spec/tasks/gitlab/gitaly_rake_spec.rb +++ b/spec/tasks/gitlab/gitaly_rake_spec.rb @@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do describe 'install' do let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' } let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s } + let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s } let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp } + subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) } + context 'no dir given' do it 'aborts and display a help message' do # avoid writing task output to spec progress allow($stderr).to receive :write - expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/ + expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/ + end + end + + context 'no storage path given' do + it 'aborts and display a help message' do + allow($stderr).to receive :write + expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/ end end @@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(main_object) .to receive(:checkout_or_clone_version).and_raise 'Git error' - expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error' + expect { subject }.to raise_error 'Git error' end end @@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(main_object) .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end @@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0]) expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end @@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do it 'calls make in the gitaly directory' do expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end context 'when Rails.env is test' do @@ -89,53 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do it 'calls make in the gitaly directory with --no-deployment flag for bundle' do expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end end end end - - describe 'storage_config' do - it 'prints storage configuration in a TOML format' do - config = { - 'default' => Gitlab::GitalyClient::StorageSettings.new( - 'path' => '/path/to/default', - 'gitaly_address' => 'unix:/path/to/my.socket' - ), - 'nfs_01' => Gitlab::GitalyClient::StorageSettings.new( - 'path' => '/path/to/nfs_01', - 'gitaly_address' => 'unix:/path/to/my.socket' - ) - } - allow(Gitlab.config.repositories).to receive(:storages).and_return(config) - allow(Rails.env).to receive(:test?).and_return(false) - - expected_output = '' - Timecop.freeze do - expected_output = <<~TOML - # Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)} - # This is in TOML format suitable for use in Gitaly's config.toml file. - bin_dir = "tmp/tests/gitaly" - socket_path = "/path/to/my.socket" - [gitlab-shell] - dir = "#{Gitlab.config.gitlab_shell.path}" - [[storage]] - name = "default" - path = "/path/to/default" - [[storage]] - name = "nfs_01" - path = "/path/to/nfs_01" - TOML - end - - expect { run_rake_task('gitlab:gitaly:storage_config')} - .to output(expected_output).to_stdout - - parsed_output = TomlRB.parse(expected_output) - config.each do |name, params| - expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path }) - end - end - end end diff --git a/spec/tasks/gitlab/shell_rake_spec.rb b/spec/tasks/gitlab/shell_rake_spec.rb index 4a756c5742d..0ed5d3e27b9 100644 --- a/spec/tasks/gitlab/shell_rake_spec.rb +++ b/spec/tasks/gitlab/shell_rake_spec.rb @@ -7,11 +7,17 @@ describe 'gitlab:shell rake tasks' do stub_warn_user_is_not_gitlab end + after do + TestEnv.create_fake_git_hooks + end + describe 'install task' do it 'invokes create_hooks task' do expect(Rake::Task['gitlab:shell:create_hooks']).to receive(:invoke) - storages = Gitlab.config.repositories.storages.values.map(&:legacy_disk_path) + storages = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.values.map(&:legacy_disk_path) + end expect(Kernel).to receive(:system).with('bin/install', *storages).and_call_original expect(Kernel).to receive(:system).with('bin/compile').and_call_original diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index 35e451b2f9a..6b50670c3c0 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -1,6 +1,6 @@ require 'rake_helper' -describe 'gitlab:storage:*' do +describe 'rake gitlab:storage:*' do before do Rake.application.rake_require 'tasks/gitlab/storage' @@ -44,28 +44,40 @@ describe 'gitlab:storage:*' do end describe 'gitlab:storage:migrate_to_hashed' do + let(:task) { 'gitlab:storage:migrate_to_hashed' } + + context 'read-only database' do + it 'does nothing' do + expect(Gitlab::Database).to receive(:read_only?).and_return(true) + + expect(Project).not_to receive(:with_unmigrated_storage) + + expect { run_rake_task(task) }.to output(/This task requires database write access. Exiting./).to_stderr + end + end + context '0 legacy projects' do it 'does nothing' do - expect(StorageMigratorWorker).not_to receive(:perform_async) + expect(::HashedStorage::MigratorWorker).not_to receive(:perform_async) - run_rake_task('gitlab:storage:migrate_to_hashed') + run_rake_task(task) end end context '3 legacy projects' do - let(:projects) { create_list(:project, 3, storage_version: 0) } + let(:projects) { create_list(:project, 3, :legacy_storage) } context 'in batches of 1' do before do stub_env('BATCH' => 1) end - it 'enqueues one StorageMigratorWorker per project' do + it 'enqueues one HashedStorage::MigratorWorker per project' do projects.each do |project| - expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id) + expect(::HashedStorage::MigratorWorker).to receive(:perform_async).with(project.id, project.id) end - run_rake_task('gitlab:storage:migrate_to_hashed') + run_rake_task(task) end end @@ -74,29 +86,54 @@ describe 'gitlab:storage:*' do stub_env('BATCH' => 2) end - it 'enqueues one StorageMigratorWorker per 2 projects' do + it 'enqueues one HashedStorage::MigratorWorker per 2 projects' do projects.map(&:id).sort.each_slice(2) do |first, last| last ||= first - expect(StorageMigratorWorker).to receive(:perform_async).with(first, last) + expect(::HashedStorage::MigratorWorker).to receive(:perform_async).with(first, last) end - run_rake_task('gitlab:storage:migrate_to_hashed') + run_rake_task(task) end end end + + context 'with same id in range' do + it 'displays message when project cant be found' do + stub_env('ID_FROM', 99999) + stub_env('ID_TO', 99999) + + expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stderr + end + + it 'displays a message when project exists but its already migrated' do + project = create(:project) + stub_env('ID_FROM', project.id) + stub_env('ID_TO', project.id) + + expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stderr + end + + it 'enqueues migration when project can be found' do + project = create(:project, :legacy_storage) + stub_env('ID_FROM', project.id) + stub_env('ID_TO', project.id) + + expect { run_rake_task(task) }.to output(/Enqueueing storage migration .* \(ID=#{project.id}\)/).to_stdout + end + end end describe 'gitlab:storage:legacy_projects' do it_behaves_like 'rake entities summary', 'projects', 'Legacy' do let(:task) { 'gitlab:storage:legacy_projects' } - let(:create_collection) { create_list(:project, 3, storage_version: 0) } + let(:create_collection) { create_list(:project, 3, :legacy_storage) } end end describe 'gitlab:storage:list_legacy_projects' do it_behaves_like 'rake listing entities', 'projects', 'Legacy' do let(:task) { 'gitlab:storage:list_legacy_projects' } - let(:create_collection) { create_list(:project, 3, storage_version: 0) } + let(:create_collection) { create_list(:project, 3, :legacy_storage) } end end @@ -133,7 +170,7 @@ describe 'gitlab:storage:*' do describe 'gitlab:storage:hashed_attachments' do it_behaves_like 'rake entities summary', 'attachments', 'Hashed' do let(:task) { 'gitlab:storage:hashed_attachments' } - let(:project) { create(:project, storage_version: 2) } + let(:project) { create(:project) } let(:create_collection) { create_list(:upload, 3, model: project) } end end @@ -141,7 +178,7 @@ describe 'gitlab:storage:*' do describe 'gitlab:storage:list_hashed_attachments' do it_behaves_like 'rake listing entities', 'attachments', 'Hashed' do let(:task) { 'gitlab:storage:list_hashed_attachments' } - let(:project) { create(:project, storage_version: 2) } + let(:project) { create(:project) } let(:create_collection) { create_list(:upload, 3, model: project) } end end diff --git a/spec/tasks/gitlab/traces_rake_spec.rb b/spec/tasks/gitlab/traces_rake_spec.rb index bd18e8ffc1e..aaf0d7242dd 100644 --- a/spec/tasks/gitlab/traces_rake_spec.rb +++ b/spec/tasks/gitlab/traces_rake_spec.rb @@ -5,51 +5,109 @@ describe 'gitlab:traces rake tasks' do Rake.application.rake_require 'tasks/gitlab/traces' end - shared_examples 'passes the job id to worker' do - it do - expect(ArchiveTraceWorker).to receive(:bulk_perform_async).with([[job.id]]) + describe 'gitlab:traces:archive' do + shared_examples 'passes the job id to worker' do + it do + expect(ArchiveTraceWorker).to receive(:bulk_perform_async).with([[job.id]]) - run_rake_task('gitlab:traces:archive') + run_rake_task('gitlab:traces:archive') + end end - end - shared_examples 'does not pass the job id to worker' do - it do - expect(ArchiveTraceWorker).not_to receive(:bulk_perform_async) + shared_examples 'does not pass the job id to worker' do + it do + expect(ArchiveTraceWorker).not_to receive(:bulk_perform_async) - run_rake_task('gitlab:traces:archive') + run_rake_task('gitlab:traces:archive') + end end - end - context 'when trace file stored in default path' do - let!(:job) { create(:ci_build, :success, :trace_live) } + context 'when trace file stored in default path' do + let!(:job) { create(:ci_build, :success, :trace_live) } - it_behaves_like 'passes the job id to worker' - end + it_behaves_like 'passes the job id to worker' + end - context 'when trace is stored in database' do - let!(:job) { create(:ci_build, :success) } + context 'when trace is stored in database' do + let!(:job) { create(:ci_build, :success) } - before do - job.update_column(:trace, 'trace in db') + before do + job.update_column(:trace, 'trace in db') + end + + it_behaves_like 'passes the job id to worker' end - it_behaves_like 'passes the job id to worker' + context 'when job has trace artifact' do + let!(:job) { create(:ci_build, :success) } + + before do + create(:ci_job_artifact, :trace, job: job) + end + + it_behaves_like 'does not pass the job id to worker' + end + + context 'when job is not finished yet' do + let!(:build) { create(:ci_build, :running, :trace_live) } + + it_behaves_like 'does not pass the job id to worker' + end end - context 'when job has trace artifact' do - let!(:job) { create(:ci_build, :success) } + describe 'gitlab:traces:migrate' do + let(:object_storage_enabled) { false } before do - create(:ci_job_artifact, :trace, job: job) + stub_artifacts_object_storage(enabled: object_storage_enabled) end - it_behaves_like 'does not pass the job id to worker' - end + subject { run_rake_task('gitlab:traces:migrate') } - context 'when job is not finished yet' do - let!(:build) { create(:ci_build, :running, :trace_live) } + let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) } - it_behaves_like 'does not pass the job id to worker' + context 'when local storage is used' do + let(:store) { ObjectStorage::Store::LOCAL } + + context 'and job does not have file store defined' do + let(:object_storage_enabled) { true } + let(:store) { nil } + + it "migrates file to remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is defined' do + let(:object_storage_enabled) { true } + + it "migrates file to remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is not defined' do + it "fails to migrate to remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL) + end + end + end + + context 'when remote storage is used' do + let(:object_storage_enabled) { true } + let(:store) { ObjectStorage::Store::REMOTE } + + it "file stays on remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end end end diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb index 6fcfae358ec..9588e8be5dc 100644 --- a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb +++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb @@ -38,14 +38,6 @@ describe 'gitlab:uploads:migrate rake tasks' do let!(:projects) { create_list(:project, 10, :with_avatar) } it_behaves_like 'enqueue jobs in batch', batch: 4 - - context 'Upload has store = nil' do - before do - Upload.where(model: projects).update_all(store: nil) - end - - it_behaves_like 'enqueue jobs in batch', batch: 4 - end end context "for Group" do diff --git a/spec/tasks/gitlab/web_hook_rake_spec.rb b/spec/tasks/gitlab/web_hook_rake_spec.rb new file mode 100644 index 00000000000..7bdf33ff6b0 --- /dev/null +++ b/spec/tasks/gitlab/web_hook_rake_spec.rb @@ -0,0 +1,92 @@ +require 'rake_helper' + +describe 'gitlab:web_hook namespace rake tasks' do + set(:group) { create(:group) } + + set(:project1) { create(:project, namespace: group) } + set(:project2) { create(:project, namespace: group) } + set(:other_group_project) { create(:project) } + + let(:url) { 'http://example.com' } + let(:hook_urls) { (project1.hooks + project2.hooks).map(&:url) } + let(:other_group_hook_urls) { other_group_project.hooks.map(&:url) } + + before do + Rake.application.rake_require 'tasks/gitlab/web_hook' + end + + describe 'gitlab:web_hook:add' do + it 'adds a web hook to all projects' do + stub_env('URL' => url) + run_rake_task('gitlab:web_hook:add') + + expect(hook_urls).to contain_exactly(url, url) + expect(other_group_hook_urls).to contain_exactly(url) + end + + it 'adds a web hook to projects in the specified namespace' do + stub_env('URL' => url, 'NAMESPACE' => group.full_path) + run_rake_task('gitlab:web_hook:add') + + expect(hook_urls).to contain_exactly(url, url) + expect(other_group_hook_urls).to be_empty + end + + it 'raises an error if an unknown namespace is specified' do + stub_env('URL' => url, 'NAMESPACE' => group.full_path) + + group.destroy + + expect { run_rake_task('gitlab:web_hook:add') }.to raise_error(SystemExit) + end + end + + describe 'gitlab:web_hook:rm' do + let!(:hook1) { create(:project_hook, project: project1, url: url) } + let!(:hook2) { create(:project_hook, project: project2, url: url) } + let!(:other_group_hook) { create(:project_hook, project: other_group_project, url: url) } + let!(:other_url_hook) { create(:project_hook, url: other_url, project: project1) } + + let(:other_url) { 'http://other.example.com' } + + it 'removes a web hook from all projects by URL' do + stub_env('URL' => url) + run_rake_task('gitlab:web_hook:rm') + + expect(hook_urls).to contain_exactly(other_url) + expect(other_group_hook_urls).to be_empty + end + + it 'removes a web hook from projects in the specified namespace by URL' do + stub_env('NAMESPACE' => group.full_path, 'URL' => url) + run_rake_task('gitlab:web_hook:rm') + + expect(hook_urls).to contain_exactly(other_url) + expect(other_group_hook_urls).to contain_exactly(url) + end + + it 'raises an error if an unknown namespace is specified' do + stub_env('URL' => url, 'NAMESPACE' => group.full_path) + + group.destroy + + expect { run_rake_task('gitlab:web_hook:rm') }.to raise_error(SystemExit) + end + end + + describe 'gitlab:web_hook:list' do + let!(:hook1) { create(:project_hook, project: project1) } + let!(:hook2) { create(:project_hook, project: project2) } + let!(:other_group_hook) { create(:project_hook, project: other_group_project) } + + it 'lists all web hooks' do + expect { run_rake_task('gitlab:web_hook:list') }.to output(/3 webhooks found/).to_stdout + end + + it 'lists web hooks in a particular namespace' do + stub_env('NAMESPACE', group.full_path) + + expect { run_rake_task('gitlab:web_hook:list') }.to output(/2 webhooks found/).to_stdout + end + end +end |