From a0808df0b627180d7773d5d13a0f64d6e7c45f5d Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Tue, 5 Jun 2018 15:51:14 +0000 Subject: Find and mark more Git disk access locations --- spec/tasks/gitlab/backup_rake_spec.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index c9252bebb2e..93a436cb2b5 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -101,7 +101,9 @@ describe 'gitlab:app namespace rake task' do before do stub_env('SKIP', 'db') - path = File.join(project.repository.path_to_repo, 'custom_hooks') + path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(project.repository.path_to_repo, 'custom_hooks') + end FileUtils.mkdir_p(path) FileUtils.touch(File.join(path, "dummy.txt")) end @@ -122,7 +124,10 @@ describe 'gitlab:app namespace rake task' do expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout - expect(Dir.entries(File.join(project.repository.path, 'custom_hooks'))).to include("dummy.txt") + repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + project.repository.path + end + expect(Dir.entries(File.join(repo_path, 'custom_hooks'))).to include("dummy.txt") end end @@ -243,10 +248,12 @@ describe 'gitlab:app namespace rake task' do FileUtils.mkdir_p(b_storage_dir) # Even when overriding the storage, we have to move it there, so it exists - FileUtils.mv( - File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'), - Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git') - ) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + FileUtils.mv( + File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'), + Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git') + ) + end expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout -- cgit v1.2.1 From 36c337647591d964b7ef1e1fc61fc64a930fb6f4 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 7 Jun 2018 15:40:44 +0000 Subject: Resolve "Hashed Storage: Make possible to migrate single project" --- spec/tasks/gitlab/storage_rake_spec.rb | 45 +++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 9 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index 35e451b2f9a..233076ad6fa 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -1,6 +1,6 @@ require 'rake_helper' -describe 'gitlab:storage:*' do +describe 'rake gitlab:storage:*' do before do Rake.application.rake_require 'tasks/gitlab/storage' @@ -44,16 +44,18 @@ describe 'gitlab:storage:*' do end describe 'gitlab:storage:migrate_to_hashed' do + let(:task) { 'gitlab:storage:migrate_to_hashed' } + context '0 legacy projects' do it 'does nothing' do expect(StorageMigratorWorker).not_to receive(:perform_async) - run_rake_task('gitlab:storage:migrate_to_hashed') + run_rake_task(task) end end context '3 legacy projects' do - let(:projects) { create_list(:project, 3, storage_version: 0) } + let(:projects) { create_list(:project, 3, :legacy_storage) } context 'in batches of 1' do before do @@ -65,7 +67,7 @@ describe 'gitlab:storage:*' do expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id) end - run_rake_task('gitlab:storage:migrate_to_hashed') + run_rake_task(task) end end @@ -80,23 +82,48 @@ describe 'gitlab:storage:*' do expect(StorageMigratorWorker).to receive(:perform_async).with(first, last) end - run_rake_task('gitlab:storage:migrate_to_hashed') + run_rake_task(task) end end end + + context 'with same id in range' do + it 'displays message when project cant be found' do + stub_env('ID_FROM', 99999) + stub_env('ID_TO', 99999) + + expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stdout + end + + it 'displays a message when project exists but its already migrated' do + project = create(:project) + stub_env('ID_FROM', project.id) + stub_env('ID_TO', project.id) + + expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stdout + end + + it 'enqueues migration when project can be found' do + project = create(:project, :legacy_storage) + stub_env('ID_FROM', project.id) + stub_env('ID_TO', project.id) + + expect { run_rake_task(task) }.to output(/Enqueueing storage migration .* \(ID=#{project.id}\)/).to_stdout + end + end end describe 'gitlab:storage:legacy_projects' do it_behaves_like 'rake entities summary', 'projects', 'Legacy' do let(:task) { 'gitlab:storage:legacy_projects' } - let(:create_collection) { create_list(:project, 3, storage_version: 0) } + let(:create_collection) { create_list(:project, 3, :legacy_storage) } end end describe 'gitlab:storage:list_legacy_projects' do it_behaves_like 'rake listing entities', 'projects', 'Legacy' do let(:task) { 'gitlab:storage:list_legacy_projects' } - let(:create_collection) { create_list(:project, 3, storage_version: 0) } + let(:create_collection) { create_list(:project, 3, :legacy_storage) } end end @@ -133,7 +160,7 @@ describe 'gitlab:storage:*' do describe 'gitlab:storage:hashed_attachments' do it_behaves_like 'rake entities summary', 'attachments', 'Hashed' do let(:task) { 'gitlab:storage:hashed_attachments' } - let(:project) { create(:project, storage_version: 2) } + let(:project) { create(:project) } let(:create_collection) { create_list(:upload, 3, model: project) } end end @@ -141,7 +168,7 @@ describe 'gitlab:storage:*' do describe 'gitlab:storage:list_hashed_attachments' do it_behaves_like 'rake listing entities', 'attachments', 'Hashed' do let(:task) { 'gitlab:storage:list_hashed_attachments' } - let(:project) { create(:project, storage_version: 2) } + let(:project) { create(:project) } let(:create_collection) { create_list(:upload, 3, model: project) } end end -- cgit v1.2.1 From f376347f24f24efc157d80de63381dd22d060630 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 5 Jun 2018 17:58:28 +0200 Subject: Find and mark more Git disk access locations, part 2 --- spec/tasks/gitlab/gitaly_rake_spec.rb | 4 +++- spec/tasks/gitlab/shell_rake_spec.rb | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb index 1e507c0236e..4545226d78c 100644 --- a/spec/tasks/gitlab/gitaly_rake_spec.rb +++ b/spec/tasks/gitlab/gitaly_rake_spec.rb @@ -134,7 +134,9 @@ describe 'gitlab:gitaly namespace rake task' do parsed_output = TomlRB.parse(expected_output) config.each do |name, params| - expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path }) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path }) + end end end end diff --git a/spec/tasks/gitlab/shell_rake_spec.rb b/spec/tasks/gitlab/shell_rake_spec.rb index 4a756c5742d..0ed5d3e27b9 100644 --- a/spec/tasks/gitlab/shell_rake_spec.rb +++ b/spec/tasks/gitlab/shell_rake_spec.rb @@ -7,11 +7,17 @@ describe 'gitlab:shell rake tasks' do stub_warn_user_is_not_gitlab end + after do + TestEnv.create_fake_git_hooks + end + describe 'install task' do it 'invokes create_hooks task' do expect(Rake::Task['gitlab:shell:create_hooks']).to receive(:invoke) - storages = Gitlab.config.repositories.storages.values.map(&:legacy_disk_path) + storages = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.values.map(&:legacy_disk_path) + end expect(Kernel).to receive(:system).with('bin/install', *storages).and_call_original expect(Kernel).to receive(:system).with('bin/compile').and_call_original -- cgit v1.2.1 From 1ef3b3efbddd6c4a81102acf940674d80e0c0a8d Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Wed, 27 Jun 2018 13:42:22 -0700 Subject: Update tests for gitlab:db:configure --- spec/tasks/gitlab/db_rake_spec.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index fc52c04e78d..81b9ac94ff3 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -20,7 +20,7 @@ describe 'gitlab:db namespace rake task' do describe 'configure' do it 'invokes db:migrate when schema has already been loaded' do - allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['default']) + allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['table1', 'table2']) expect(Rake::Task['db:migrate']).to receive(:invoke) expect(Rake::Task['db:schema:load']).not_to receive(:invoke) expect(Rake::Task['db:seed_fu']).not_to receive(:invoke) @@ -35,6 +35,14 @@ describe 'gitlab:db namespace rake task' do expect { run_rake_task('gitlab:db:configure') }.not_to raise_error end + it 'invokes db:shema:load and db:seed_fu when there is only a single table present' do + allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['default']) + expect(Rake::Task['db:schema:load']).to receive(:invoke) + expect(Rake::Task['db:seed_fu']).to receive(:invoke) + expect(Rake::Task['db:migrate']).not_to receive(:invoke) + expect { run_rake_task('gitlab:db:configure') }.not_to raise_error + end + it 'does not invoke any other rake tasks during an error' do allow(ActiveRecord::Base).to receive(:connection).and_raise(RuntimeError, 'error') expect(Rake::Task['db:migrate']).not_to receive(:invoke) -- cgit v1.2.1 From 2efe4a13b6be650258ade395438600c64820cb19 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Wed, 27 Jun 2018 14:09:06 -0700 Subject: Fix static analysis failure --- spec/tasks/gitlab/db_rake_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index 81b9ac94ff3..b81aea23306 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -20,7 +20,7 @@ describe 'gitlab:db namespace rake task' do describe 'configure' do it 'invokes db:migrate when schema has already been loaded' do - allow(ActiveRecord::Base.connection).to receive(:tables).and_return(['table1', 'table2']) + allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[table1 table2]) expect(Rake::Task['db:migrate']).to receive(:invoke) expect(Rake::Task['db:schema:load']).not_to receive(:invoke) expect(Rake::Task['db:seed_fu']).not_to receive(:invoke) -- cgit v1.2.1 From f5da4815a50ee3551c7330b85dc3c209e0fd0d57 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 3 Jul 2018 17:20:51 -0500 Subject: Speed up spec/tasks/gitlab/git_rake_spec.rb Because no Git repository was actually created at the temporary path we were using, `git fsck` would traverse up until it found a repository, which in our case was the CE or EE repository. --- spec/tasks/gitlab/git_rake_spec.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb index 1efaecc63a5..d0263ad9a37 100644 --- a/spec/tasks/gitlab/git_rake_spec.rb +++ b/spec/tasks/gitlab/git_rake_spec.rb @@ -1,15 +1,20 @@ require 'rake_helper' describe 'gitlab:git rake tasks' do + let(:base_path) { 'tmp/tests/default_storage' } + before(:all) do @default_storage_hash = Gitlab.config.repositories.storages.default.to_h end before do Rake.application.rake_require 'tasks/gitlab/git' - storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) } + storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => base_path)) } + + path = Settings.absolute("#{base_path}/@hashed/1/2/test.git") + FileUtils.mkdir_p(path) + Gitlab::Popen.popen(%W[git -C #{path} init --bare]) - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git')) allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) allow_any_instance_of(String).to receive(:color) { |string, _color| string } @@ -17,7 +22,7 @@ describe 'gitlab:git rake tasks' do end after do - FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage')) + FileUtils.rm_rf(Settings.absolute(base_path)) end describe 'fsck' do @@ -26,14 +31,14 @@ describe 'gitlab:git rake tasks' do end it 'errors out about config.lock issues' do - FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/config.lock')) + FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/config.lock")) expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout end it 'errors out about ref lock issues' do - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads')) - FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads/blah.lock')) + FileUtils.mkdir_p(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads")) + FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads/blah.lock")) expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout end -- cgit v1.2.1 From f1f7bfc06f506469c5d469ba91350ff57780f117 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 24 Jul 2018 12:17:29 +0200 Subject: Remove git rake tasks These tasks are happening through housekeeping right now, by default ever 10th push. This removes the need for these tasks. Side note, this removes one of my first contributions to GitLab, as back than I introduced these tasks through: 54e6c0045bb13c05cc5478cbdf47d3246bd9fe2b Closes https://gitlab.com/gitlab-org/gitaly/issues/768 --- spec/tasks/gitlab/git_rake_spec.rb | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb index d0263ad9a37..57b006e1a39 100644 --- a/spec/tasks/gitlab/git_rake_spec.rb +++ b/spec/tasks/gitlab/git_rake_spec.rb @@ -2,45 +2,19 @@ require 'rake_helper' describe 'gitlab:git rake tasks' do let(:base_path) { 'tmp/tests/default_storage' } - - before(:all) do - @default_storage_hash = Gitlab.config.repositories.storages.default.to_h - end + let!(:project) { create(:project, :repository) } before do Rake.application.rake_require 'tasks/gitlab/git' - storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => base_path)) } - - path = Settings.absolute("#{base_path}/@hashed/1/2/test.git") - FileUtils.mkdir_p(path) - Gitlab::Popen.popen(%W[git -C #{path} init --bare]) - allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) allow_any_instance_of(String).to receive(:color) { |string, _color| string } stub_warn_user_is_not_gitlab end - after do - FileUtils.rm_rf(Settings.absolute(base_path)) - end - describe 'fsck' do it 'outputs the integrity check for a repo' do - expect { run_rake_task('gitlab:git:fsck') }.to output(%r{Performed Checking integrity at .*@hashed/1/2/test.git}).to_stdout - end - - it 'errors out about config.lock issues' do - FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/config.lock")) - - expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout - end - - it 'errors out about ref lock issues' do - FileUtils.mkdir_p(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads")) - FileUtils.touch(Settings.absolute("#{base_path}/@hashed/1/2/test.git/refs/heads/blah.lock")) - - expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout + expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed integrity check for/).to_stdout end end end -- cgit v1.2.1 From d7afed34c4238d637ce71fe8dbea41a64e7a3f1f Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 26 Jul 2018 10:05:32 +0200 Subject: Remove feature flags from lib/backup Moved to OPT_OUT in 7d14b725a0da41d1ae7c0a8496b5e66832023e3b, Now, by removing the feature gates, this is an mandatory feature. Related issues: - https://gitlab.com/gitlab-org/gitaly/issues/526 - https://gitlab.com/gitlab-org/gitaly/issues/1194 Closes https://gitlab.com/gitlab-org/gitaly/issues/749 Closes https://gitlab.com/gitlab-org/gitaly/issues/1212 Closes https://gitlab.com/gitlab-org/gitaly/issues/1195 --- spec/tasks/gitlab/backup_rake_spec.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 93a436cb2b5..3ba6caf1337 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -87,6 +87,27 @@ describe 'gitlab:app namespace rake task' do expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout end end + + context 'when the restore directory is not empty' do + before do + # We only need a backup of the repositories for this test + stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry') + end + + it 'removes stale data' do + expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout + + excluded_project = create(:project, :repository, name: 'mepmep') + + expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout + + raw_repo = excluded_project.repository.raw + + # The restore will not find the repository in the backup, but will create + # an empty one in its place + expect(raw_repo.empty?).to be(true) + end + end end # backup_restore task describe 'backup' do -- cgit v1.2.1 From 3cbd8b13436be25074db17dfbd555588e917279d Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 26 Jul 2018 14:23:33 -0700 Subject: Add local project uploads cleanup task --- spec/tasks/gitlab/cleanup_rake_spec.rb | 317 ++++++++++++++++++++++++++++++++- 1 file changed, 316 insertions(+), 1 deletion(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb index 2bf873c923f..ba08ece1b4b 100644 --- a/spec/tasks/gitlab/cleanup_rake_spec.rb +++ b/spec/tasks/gitlab/cleanup_rake_spec.rb @@ -5,7 +5,7 @@ describe 'gitlab:cleanup rake tasks' do Rake.application.rake_require 'tasks/gitlab/cleanup' end - describe 'cleanup' do + describe 'cleanup namespaces and repos' do let(:storages) do { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) @@ -67,4 +67,319 @@ describe 'gitlab:cleanup rake tasks' do end end end + + describe 'cleanup:project_uploads' do + context 'orphaned project upload file' do + context 'when an upload record matching the secret and filename is found' do + context 'when the project is still in legacy storage' do + let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) } + let!(:correct_path) { orphaned.absolute_path } + let!(:other_project) { create(:project, :legacy_storage) } + let!(:orphaned_path) { correct_path.sub(/#{orphaned.model.full_path}/, other_project.full_path) } + + before do + FileUtils.mkdir_p(File.dirname(orphaned_path)) + FileUtils.mv(correct_path, orphaned_path) + end + + it 'moves the file to its proper location' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}") + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(correct_path)).to be_falsey + + stub_env('DRY_RUN', 'false') + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_falsey + expect(File.exist?(correct_path)).to be_truthy + end + + it 'a dry run does not move the file' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}") + expect(Rails.logger).to receive(:info) + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(correct_path)).to be_falsey + + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(correct_path)).to be_falsey + end + + context 'when the project record is missing (Upload#absolute_path raises error)' do + let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', other_project.full_path, orphaned.path) } + + before do + orphaned.model.delete + end + + it 'moves the file to lost and found' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}") + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + + stub_env('DRY_RUN', 'false') + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_falsey + expect(File.exist?(lost_and_found_path)).to be_truthy + end + + it 'a dry run does not move the file' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}") + expect(Rails.logger).to receive(:info) + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + end + end + end + + context 'when the project was moved to hashed storage' do + let!(:orphaned) { create(:upload, :issuable_upload, :with_file) } + let!(:correct_path) { orphaned.absolute_path } + let!(:orphaned_path) { File.join(FileUploader.root, 'foo', 'bar', orphaned.path) } + + before do + FileUtils.mkdir_p(File.dirname(orphaned_path)) + FileUtils.mv(correct_path, orphaned_path) + end + + it 'moves the file to its proper location' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}") + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(correct_path)).to be_falsey + + stub_env('DRY_RUN', 'false') + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_falsey + expect(File.exist?(correct_path)).to be_truthy + end + + it 'a dry run does not move the file' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}") + expect(Rails.logger).to receive(:info) + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(correct_path)).to be_falsey + + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(correct_path)).to be_falsey + end + end + end + + context 'when a matching upload record can not be found' do + context 'when the file path fits the known pattern' do + let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) } + let!(:orphaned_path) { orphaned.absolute_path } + let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', orphaned.model.full_path, orphaned.path) } + + before do + orphaned.delete + end + + it 'moves the file to lost and found' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}") + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + + stub_env('DRY_RUN', 'false') + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_falsey + expect(File.exist?(lost_and_found_path)).to be_truthy + end + + it 'a dry run does not move the file' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}") + expect(Rails.logger).to receive(:info) + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + end + end + + context 'when the file path does not fit the known pattern' do + let!(:invalid_path) { File.join('group', 'file.jpg') } + let!(:orphaned_path) { File.join(FileUploader.root, invalid_path) } + let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', invalid_path) } + + before do + FileUtils.mkdir_p(File.dirname(orphaned_path)) + FileUtils.touch(orphaned_path) + end + + after do + File.delete(orphaned_path) if File.exist?(orphaned_path) + end + + it 'moves the file to lost and found' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}") + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + + stub_env('DRY_RUN', 'false') + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_falsey + expect(File.exist?(lost_and_found_path)).to be_truthy + end + + it 'a dry run does not move the file' do + expect(Rails.logger).to receive(:info).twice + expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}") + expect(Rails.logger).to receive(:info) + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_truthy + expect(File.exist?(lost_and_found_path)).to be_falsey + end + end + end + end + + context 'non-orphaned project upload file' do + it 'does not move the file' do + tracked = create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) + tracked_path = tracked.absolute_path + + expect(Rails.logger).not_to receive(:info).with(/move|fix/i) + expect(File.exist?(tracked_path)).to be_truthy + + stub_env('DRY_RUN', 'false') + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(tracked_path)).to be_truthy + end + end + + context 'ignorable cases' do + shared_examples_for 'does not move anything' do + it 'does not move even an orphan file' do + orphaned = create(:upload, :issuable_upload, :with_file, model: project) + orphaned_path = orphaned.absolute_path + orphaned.delete + + expect(File.exist?(orphaned_path)).to be_truthy + + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(orphaned_path)).to be_truthy + end + end + + # Because we aren't concerned about these, and can save a lot of + # processing time by ignoring them. If we wish to cleanup hashed storage + # directories, it should simply require removing this test and modifying + # the find command. + context 'when the file is already in hashed storage' do + let(:project) { create(:project) } + + before do + stub_env('DRY_RUN', 'false') + expect(Rails.logger).not_to receive(:info).with(/move|fix/i) + end + + it_behaves_like 'does not move anything' + end + + context 'when DRY_RUN env var is unset' do + let(:project) { create(:project, :legacy_storage) } + + it_behaves_like 'does not move anything' + end + + context 'when DRY_RUN env var is true' do + let(:project) { create(:project, :legacy_storage) } + + before do + stub_env('DRY_RUN', 'true') + end + + it_behaves_like 'does not move anything' + end + + context 'when DRY_RUN env var is foo' do + let(:project) { create(:project, :legacy_storage) } + + before do + stub_env('DRY_RUN', 'foo') + end + + it_behaves_like 'does not move anything' + end + + it 'does not move any non-project (FileUploader) uploads' do + stub_env('DRY_RUN', 'false') + + paths = [] + orphaned1 = create(:upload, :personal_snippet_upload, :with_file) + orphaned2 = create(:upload, :namespace_upload, :with_file) + orphaned3 = create(:upload, :attachment_upload, :with_file) + paths << orphaned1.absolute_path + paths << orphaned2.absolute_path + paths << orphaned3.absolute_path + Upload.delete_all + + expect(Rails.logger).not_to receive(:info).with(/move|fix/i) + paths.each do |path| + expect(File.exist?(path)).to be_truthy + end + + run_rake_task('gitlab:cleanup:project_uploads') + + paths.each do |path| + expect(File.exist?(path)).to be_truthy + end + end + + it 'does not move any uploads in tmp (which would interfere with ongoing upload activity)' do + stub_env('DRY_RUN', 'false') + + path = File.join(FileUploader.root, 'tmp', 'foo.jpg') + FileUtils.mkdir_p(File.dirname(path)) + FileUtils.touch(path) + + expect(Rails.logger).not_to receive(:info).with(/move|fix/i) + expect(File.exist?(path)).to be_truthy + + run_rake_task('gitlab:cleanup:project_uploads') + + expect(File.exist?(path)).to be_truthy + end + end + end end -- cgit v1.2.1 From a4351ac077b9f266b4bcfa8f1c4867a837870a27 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 3 Aug 2018 04:36:43 +0000 Subject: Add object storage related tests for `gitlab:cleanup:project_uploads` task --- spec/tasks/gitlab/cleanup_rake_spec.rb | 321 +++++---------------------------- 1 file changed, 45 insertions(+), 276 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb index ba08ece1b4b..cc2cca10f58 100644 --- a/spec/tasks/gitlab/cleanup_rake_spec.rb +++ b/spec/tasks/gitlab/cleanup_rake_spec.rb @@ -68,317 +68,86 @@ describe 'gitlab:cleanup rake tasks' do end end + # A single integration test that is redundant with one part of the + # Gitlab::Cleanup::ProjectUploads spec. + # + # Additionally, this tests DRY_RUN env var values, and the extra line of + # output that says you can disable DRY_RUN if it's enabled. describe 'cleanup:project_uploads' do - context 'orphaned project upload file' do - context 'when an upload record matching the secret and filename is found' do - context 'when the project is still in legacy storage' do - let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) } - let!(:correct_path) { orphaned.absolute_path } - let!(:other_project) { create(:project, :legacy_storage) } - let!(:orphaned_path) { correct_path.sub(/#{orphaned.model.full_path}/, other_project.full_path) } + let!(:logger) { double(:logger) } - before do - FileUtils.mkdir_p(File.dirname(orphaned_path)) - FileUtils.mv(correct_path, orphaned_path) - end - - it 'moves the file to its proper location' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}") - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(correct_path)).to be_falsey - - stub_env('DRY_RUN', 'false') - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_falsey - expect(File.exist?(correct_path)).to be_truthy - end - - it 'a dry run does not move the file' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}") - expect(Rails.logger).to receive(:info) - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(correct_path)).to be_falsey - - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(correct_path)).to be_falsey - end - - context 'when the project record is missing (Upload#absolute_path raises error)' do - let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', other_project.full_path, orphaned.path) } - - before do - orphaned.model.delete - end - - it 'moves the file to lost and found' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}") - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - - stub_env('DRY_RUN', 'false') - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_falsey - expect(File.exist?(lost_and_found_path)).to be_truthy - end - - it 'a dry run does not move the file' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}") - expect(Rails.logger).to receive(:info) - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - end - end - end - - context 'when the project was moved to hashed storage' do - let!(:orphaned) { create(:upload, :issuable_upload, :with_file) } - let!(:correct_path) { orphaned.absolute_path } - let!(:orphaned_path) { File.join(FileUploader.root, 'foo', 'bar', orphaned.path) } - - before do - FileUtils.mkdir_p(File.dirname(orphaned_path)) - FileUtils.mv(correct_path, orphaned_path) - end - - it 'moves the file to its proper location' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}") - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(correct_path)).to be_falsey - - stub_env('DRY_RUN', 'false') - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_falsey - expect(File.exist?(correct_path)).to be_truthy - end - - it 'a dry run does not move the file' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}") - expect(Rails.logger).to receive(:info) + before do + expect(main_object).to receive(:logger).and_return(logger).at_least(1).times - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(correct_path)).to be_falsey + allow(logger).to receive(:info).at_least(1).times + allow(logger).to receive(:debug).at_least(1).times + end - run_rake_task('gitlab:cleanup:project_uploads') + context 'with a fixable orphaned project upload file' do + let(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) } + let(:new_path) { orphaned.absolute_path } + let(:path) { File.join(FileUploader.root, 'some', 'wrong', 'location', orphaned.path) } - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(correct_path)).to be_falsey - end - end + before do + FileUtils.mkdir_p(File.dirname(path)) + FileUtils.mv(new_path, path) end - context 'when a matching upload record can not be found' do - context 'when the file path fits the known pattern' do - let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) } - let!(:orphaned_path) { orphaned.absolute_path } - let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', orphaned.model.full_path, orphaned.path) } - - before do - orphaned.delete - end - - it 'moves the file to lost and found' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}") - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - - stub_env('DRY_RUN', 'false') - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_falsey - expect(File.exist?(lost_and_found_path)).to be_truthy - end - - it 'a dry run does not move the file' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}") - expect(Rails.logger).to receive(:info) - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - end + context 'with DRY_RUN disabled' do + before do + stub_env('DRY_RUN', 'false') end - context 'when the file path does not fit the known pattern' do - let!(:invalid_path) { File.join('group', 'file.jpg') } - let!(:orphaned_path) { File.join(FileUploader.root, invalid_path) } - let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', invalid_path) } - - before do - FileUtils.mkdir_p(File.dirname(orphaned_path)) - FileUtils.touch(orphaned_path) - end - - after do - File.delete(orphaned_path) if File.exist?(orphaned_path) - end - - it 'moves the file to lost and found' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}") - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - - stub_env('DRY_RUN', 'false') - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_falsey - expect(File.exist?(lost_and_found_path)).to be_truthy - end - - it 'a dry run does not move the file' do - expect(Rails.logger).to receive(:info).twice - expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}") - expect(Rails.logger).to receive(:info) - - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - - run_rake_task('gitlab:cleanup:project_uploads') + it 'moves the file to its proper location' do + run_rake_task('gitlab:cleanup:project_uploads') - expect(File.exist?(orphaned_path)).to be_truthy - expect(File.exist?(lost_and_found_path)).to be_falsey - end + expect(File.exist?(path)).to be_falsey + expect(File.exist?(new_path)).to be_truthy end - end - end - - context 'non-orphaned project upload file' do - it 'does not move the file' do - tracked = create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) - tracked_path = tracked.absolute_path - expect(Rails.logger).not_to receive(:info).with(/move|fix/i) - expect(File.exist?(tracked_path)).to be_truthy - - stub_env('DRY_RUN', 'false') - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(tracked_path)).to be_truthy - end - end - - context 'ignorable cases' do - shared_examples_for 'does not move anything' do - it 'does not move even an orphan file' do - orphaned = create(:upload, :issuable_upload, :with_file, model: project) - orphaned_path = orphaned.absolute_path - orphaned.delete - - expect(File.exist?(orphaned_path)).to be_truthy + it 'logs action as done' do + expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up...") + expect(logger).to receive(:info).with("Did fix #{path} -> #{new_path}") run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(orphaned_path)).to be_truthy end end - # Because we aren't concerned about these, and can save a lot of - # processing time by ignoring them. If we wish to cleanup hashed storage - # directories, it should simply require removing this test and modifying - # the find command. - context 'when the file is already in hashed storage' do - let(:project) { create(:project) } + shared_examples_for 'does not move the file' do + it 'does not move the file' do + run_rake_task('gitlab:cleanup:project_uploads') - before do - stub_env('DRY_RUN', 'false') - expect(Rails.logger).not_to receive(:info).with(/move|fix/i) + expect(File.exist?(path)).to be_truthy + expect(File.exist?(new_path)).to be_falsey end - it_behaves_like 'does not move anything' - end - - context 'when DRY_RUN env var is unset' do - let(:project) { create(:project, :legacy_storage) } + it 'logs action as able to be done' do + expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up. Dry run...") + expect(logger).to receive(:info).with("Can fix #{path} -> #{new_path}") + expect(logger).to receive(:info).with(/To clean up these files run this command with DRY_RUN=false/) - it_behaves_like 'does not move anything' + run_rake_task('gitlab:cleanup:project_uploads') + end end - context 'when DRY_RUN env var is true' do - let(:project) { create(:project, :legacy_storage) } - + context 'with DRY_RUN explicitly enabled' do before do stub_env('DRY_RUN', 'true') end - it_behaves_like 'does not move anything' + it_behaves_like 'does not move the file' end - context 'when DRY_RUN env var is foo' do - let(:project) { create(:project, :legacy_storage) } - + context 'with DRY_RUN set to an unknown value' do before do stub_env('DRY_RUN', 'foo') end - it_behaves_like 'does not move anything' + it_behaves_like 'does not move the file' end - it 'does not move any non-project (FileUploader) uploads' do - stub_env('DRY_RUN', 'false') - - paths = [] - orphaned1 = create(:upload, :personal_snippet_upload, :with_file) - orphaned2 = create(:upload, :namespace_upload, :with_file) - orphaned3 = create(:upload, :attachment_upload, :with_file) - paths << orphaned1.absolute_path - paths << orphaned2.absolute_path - paths << orphaned3.absolute_path - Upload.delete_all - - expect(Rails.logger).not_to receive(:info).with(/move|fix/i) - paths.each do |path| - expect(File.exist?(path)).to be_truthy - end - - run_rake_task('gitlab:cleanup:project_uploads') - - paths.each do |path| - expect(File.exist?(path)).to be_truthy - end - end - - it 'does not move any uploads in tmp (which would interfere with ongoing upload activity)' do - stub_env('DRY_RUN', 'false') - - path = File.join(FileUploader.root, 'tmp', 'foo.jpg') - FileUtils.mkdir_p(File.dirname(path)) - FileUtils.touch(path) - - expect(Rails.logger).not_to receive(:info).with(/move|fix/i) - expect(File.exist?(path)).to be_truthy - - run_rake_task('gitlab:cleanup:project_uploads') - - expect(File.exist?(path)).to be_truthy + context 'with DRY_RUN unset' do + it_behaves_like 'does not move the file' end end end -- cgit v1.2.1 From eb1a3798adb3836e4ebe641c8eb962b6f7220004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Thu, 9 Aug 2018 01:08:09 -0400 Subject: Remove storage path dependency of gitaly install task --- spec/tasks/gitlab/gitaly_rake_spec.rb | 67 +++++++++-------------------------- 1 file changed, 16 insertions(+), 51 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb index 4545226d78c..e6e4d9504d9 100644 --- a/spec/tasks/gitlab/gitaly_rake_spec.rb +++ b/spec/tasks/gitlab/gitaly_rake_spec.rb @@ -8,13 +8,23 @@ describe 'gitlab:gitaly namespace rake task' do describe 'install' do let(:repo) { 'https://gitlab.com/gitlab-org/gitaly.git' } let(:clone_path) { Rails.root.join('tmp/tests/gitaly').to_s } + let(:storage_path) { Rails.root.join('tmp/tests/repositories').to_s } let(:version) { File.read(Rails.root.join(Gitlab::GitalyClient::SERVER_VERSION_FILE)).chomp } + subject { run_rake_task('gitlab:gitaly:install', clone_path, storage_path) } + context 'no dir given' do it 'aborts and display a help message' do # avoid writing task output to spec progress allow($stderr).to receive :write - expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly/ + expect { run_rake_task('gitlab:gitaly:install') }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/ + end + end + + context 'no storage path given' do + it 'aborts and display a help message' do + allow($stderr).to receive :write + expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error /Please specify the directory where you want to install gitaly and the path for the default storage/ end end @@ -23,7 +33,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(main_object) .to receive(:checkout_or_clone_version).and_raise 'Git error' - expect { run_rake_task('gitlab:gitaly:install', clone_path) }.to raise_error 'Git error' + expect { subject }.to raise_error 'Git error' end end @@ -36,7 +46,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(main_object) .to receive(:checkout_or_clone_version).with(version: version, repo: repo, target_dir: clone_path) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end @@ -59,7 +69,7 @@ describe 'gitlab:gitaly namespace rake task' do expect(Gitlab::Popen).to receive(:popen).with(%w[which gmake]).and_return(['/usr/bin/gmake', 0]) expect(main_object).to receive(:run_command!).with(command_preamble + %w[gmake]).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end @@ -72,7 +82,7 @@ describe 'gitlab:gitaly namespace rake task' do it 'calls make in the gitaly directory' do expect(main_object).to receive(:run_command!).with(command_preamble + %w[make]).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end context 'when Rails.env is test' do @@ -89,55 +99,10 @@ describe 'gitlab:gitaly namespace rake task' do it 'calls make in the gitaly directory with --no-deployment flag for bundle' do expect(main_object).to receive(:run_command!).with(command_preamble + command).and_return(true) - run_rake_task('gitlab:gitaly:install', clone_path) + subject end end end end end - - describe 'storage_config' do - it 'prints storage configuration in a TOML format' do - config = { - 'default' => Gitlab::GitalyClient::StorageSettings.new( - 'path' => '/path/to/default', - 'gitaly_address' => 'unix:/path/to/my.socket' - ), - 'nfs_01' => Gitlab::GitalyClient::StorageSettings.new( - 'path' => '/path/to/nfs_01', - 'gitaly_address' => 'unix:/path/to/my.socket' - ) - } - allow(Gitlab.config.repositories).to receive(:storages).and_return(config) - allow(Rails.env).to receive(:test?).and_return(false) - - expected_output = '' - Timecop.freeze do - expected_output = <<~TOML - # Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)} - # This is in TOML format suitable for use in Gitaly's config.toml file. - bin_dir = "tmp/tests/gitaly" - socket_path = "/path/to/my.socket" - [gitlab-shell] - dir = "#{Gitlab.config.gitlab_shell.path}" - [[storage]] - name = "default" - path = "/path/to/default" - [[storage]] - name = "nfs_01" - path = "/path/to/nfs_01" - TOML - end - - expect { run_rake_task('gitlab:gitaly:storage_config')} - .to output(expected_output).to_stdout - - parsed_output = TomlRB.parse(expected_output) - config.each do |name, params| - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path }) - end - end - end - end end -- cgit v1.2.1 From 63091cfe6432b2290f6ccd0c1e8105c8900b9df5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 16 Aug 2018 14:28:47 +0000 Subject: Add rake command to migrate archived traces from local storage to object storage --- spec/tasks/gitlab/traces_rake_spec.rb | 112 ++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 27 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/traces_rake_spec.rb b/spec/tasks/gitlab/traces_rake_spec.rb index bd18e8ffc1e..aaf0d7242dd 100644 --- a/spec/tasks/gitlab/traces_rake_spec.rb +++ b/spec/tasks/gitlab/traces_rake_spec.rb @@ -5,51 +5,109 @@ describe 'gitlab:traces rake tasks' do Rake.application.rake_require 'tasks/gitlab/traces' end - shared_examples 'passes the job id to worker' do - it do - expect(ArchiveTraceWorker).to receive(:bulk_perform_async).with([[job.id]]) + describe 'gitlab:traces:archive' do + shared_examples 'passes the job id to worker' do + it do + expect(ArchiveTraceWorker).to receive(:bulk_perform_async).with([[job.id]]) - run_rake_task('gitlab:traces:archive') + run_rake_task('gitlab:traces:archive') + end end - end - shared_examples 'does not pass the job id to worker' do - it do - expect(ArchiveTraceWorker).not_to receive(:bulk_perform_async) + shared_examples 'does not pass the job id to worker' do + it do + expect(ArchiveTraceWorker).not_to receive(:bulk_perform_async) - run_rake_task('gitlab:traces:archive') + run_rake_task('gitlab:traces:archive') + end end - end - context 'when trace file stored in default path' do - let!(:job) { create(:ci_build, :success, :trace_live) } + context 'when trace file stored in default path' do + let!(:job) { create(:ci_build, :success, :trace_live) } - it_behaves_like 'passes the job id to worker' - end + it_behaves_like 'passes the job id to worker' + end - context 'when trace is stored in database' do - let!(:job) { create(:ci_build, :success) } + context 'when trace is stored in database' do + let!(:job) { create(:ci_build, :success) } - before do - job.update_column(:trace, 'trace in db') + before do + job.update_column(:trace, 'trace in db') + end + + it_behaves_like 'passes the job id to worker' end - it_behaves_like 'passes the job id to worker' + context 'when job has trace artifact' do + let!(:job) { create(:ci_build, :success) } + + before do + create(:ci_job_artifact, :trace, job: job) + end + + it_behaves_like 'does not pass the job id to worker' + end + + context 'when job is not finished yet' do + let!(:build) { create(:ci_build, :running, :trace_live) } + + it_behaves_like 'does not pass the job id to worker' + end end - context 'when job has trace artifact' do - let!(:job) { create(:ci_build, :success) } + describe 'gitlab:traces:migrate' do + let(:object_storage_enabled) { false } before do - create(:ci_job_artifact, :trace, job: job) + stub_artifacts_object_storage(enabled: object_storage_enabled) end - it_behaves_like 'does not pass the job id to worker' - end + subject { run_rake_task('gitlab:traces:migrate') } - context 'when job is not finished yet' do - let!(:build) { create(:ci_build, :running, :trace_live) } + let!(:job_trace) { create(:ci_job_artifact, :trace, file_store: store) } - it_behaves_like 'does not pass the job id to worker' + context 'when local storage is used' do + let(:store) { ObjectStorage::Store::LOCAL } + + context 'and job does not have file store defined' do + let(:object_storage_enabled) { true } + let(:store) { nil } + + it "migrates file to remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is defined' do + let(:object_storage_enabled) { true } + + it "migrates file to remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is not defined' do + it "fails to migrate to remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::LOCAL) + end + end + end + + context 'when remote storage is used' do + let(:object_storage_enabled) { true } + let(:store) { ObjectStorage::Store::REMOTE } + + it "file stays on remote storage" do + subject + + expect(job_trace.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end end end -- cgit v1.2.1 From 1a54986c166fb13a6a27afcafaa055e1a1749e38 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 15 Aug 2018 01:40:29 +0200 Subject: Refactor SiteStatistics to extract refresh logic into a rake task --- spec/tasks/gitlab/site_statistics_rake_spec.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 spec/tasks/gitlab/site_statistics_rake_spec.rb (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/site_statistics_rake_spec.rb b/spec/tasks/gitlab/site_statistics_rake_spec.rb new file mode 100644 index 00000000000..20f0df65e63 --- /dev/null +++ b/spec/tasks/gitlab/site_statistics_rake_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +require 'rake_helper' + +describe 'rake gitlab:refresh_site_statistics' do + before do + Rake.application.rake_require 'tasks/gitlab/site_statistics' + + create(:project) + SiteStatistic.fetch.update(repositories_count: 0, wikis_count: 0) + end + + let(:task) { 'gitlab:refresh_site_statistics' } + + it 'recalculates existing counters' do + run_rake_task(task) + + expect(SiteStatistic.fetch.repositories_count).to eq(1) + expect(SiteStatistic.fetch.wikis_count).to eq(1) + end + + it 'displays message listing counters' do + expect { run_rake_task(task) }.to output(/Updating Site Statistics counters:.* Repositories\.\.\. OK!.* Wikis\.\.\. OK!/m).to_stdout + end +end -- cgit v1.2.1 From 3aedccb17a5dbee40b5a08014c92cab8ea11e9fb Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 7 Sep 2018 11:16:34 +0200 Subject: Port cleanup tasks to use Gitaly Rake tasks cleaning up the Git storage were still using direct disk access, which won't work if these aren't attached. To mitigate a migration issue was created. To port gitlab:cleanup:dirs, and gitlab:cleanup:repos, a new RPC was required, ListDirectories. This was implemented in Gitaly, through https://gitlab.com/gitlab-org/gitaly/merge_requests/868. To be able to use the new RPC the Gitaly server was bumped to v0.120. This is an RPC that will not use feature gates, as this doesn't scale on .com so there is no way to test it at scale. Futhermore, we _know_ it doesn't scale, but this might be a useful task for smaller instances. Lastly, the tests are slightly updated to also work when the disk isn't attached. Eventhough this is not planned, it was very little effort and thus I applied the boy scout rule. Closes https://gitlab.com/gitlab-org/gitaly/issues/954 Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/40529 --- spec/tasks/gitlab/cleanup_rake_spec.rb | 35 +++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb index cc2cca10f58..19794227d9f 100644 --- a/spec/tasks/gitlab/cleanup_rake_spec.rb +++ b/spec/tasks/gitlab/cleanup_rake_spec.rb @@ -6,6 +6,8 @@ describe 'gitlab:cleanup rake tasks' do end describe 'cleanup namespaces and repos' do + let(:gitlab_shell) { Gitlab::Shell.new } + let(:storage) { storages.keys.first } let(:storages) do { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) @@ -17,53 +19,56 @@ describe 'gitlab:cleanup rake tasks' do end before do - FileUtils.mkdir(Settings.absolute('tmp/tests/default_storage')) allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) end after do - FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage')) + Gitlab::GitalyClient::StorageService.new(storage).delete_all_repositories end describe 'cleanup:repos' do before do - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/broken/project.git')) - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git')) + gitlab_shell.add_namespace(storage, 'broken/project.git') + gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git') end it 'moves it to an orphaned path' do - run_rake_task('gitlab:cleanup:repos') - repo_list = Dir['tmp/tests/default_storage/broken/*'] + now = Time.now + + Timecop.freeze(now) do + run_rake_task('gitlab:cleanup:repos') + repo_list = Gitlab::GitalyClient::StorageService.new(storage).list_directories(depth: 0) - expect(repo_list.first).to include('+orphaned+') + expect(repo_list.last).to include("broken+orphaned+#{now.to_i}") + end end it 'ignores @hashed repos' do run_rake_task('gitlab:cleanup:repos') - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy + expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true) end end describe 'cleanup:dirs' do it 'removes missing namespaces' do - FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_1/project.git")) - FileUtils.mkdir_p(Settings.absolute("tmp/tests/default_storage/namespace_2/project.git")) - allow(Namespace).to receive(:pluck).and_return('namespace_1') + gitlab_shell.add_namespace(storage, "namespace_1/project.git") + gitlab_shell.add_namespace(storage, "namespace_2/project.git") + allow(Namespace).to receive(:pluck).and_return(['namespace_1']) stub_env('REMOVE', 'true') run_rake_task('gitlab:cleanup:dirs') - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_1'))).to be_truthy - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/namespace_2'))).to be_falsey + expect(gitlab_shell.exists?(storage, 'namespace_1')).to be(true) + expect(gitlab_shell.exists?(storage, 'namespace_2')).to be(false) end it 'ignores @hashed directory' do - FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git')) + gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git') run_rake_task('gitlab:cleanup:dirs') - expect(Dir.exist?(Settings.absolute('tmp/tests/default_storage/@hashed/12/34/5678.git'))).to be_truthy + expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true) end end end -- cgit v1.2.1 From 76cfe4f1fdadf0b4e43d066ffccbe34fbfcb9a80 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Tue, 11 Sep 2018 18:41:14 -0700 Subject: Ensure the schema is loaded with post_migrations included If doing a schema load, the post_migrations should also be marked as up, even if SKIP_POST_DEPLOYMENT_MIGRATIONS was set, otherwise future migration runs will be broken. --- spec/tasks/gitlab/db_rake_spec.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index b81aea23306..5e0300066d1 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -61,6 +61,35 @@ describe 'gitlab:db namespace rake task' do expect(Rake::Task['db:migrate']).not_to receive(:invoke) expect { run_rake_task('gitlab:db:configure') }.to raise_error(RuntimeError, 'error') end + + context 'SKIP_POST_DEPLOYMENT_MIGRATIONS environment variable set' do + let :migrations_paths do + root = Rails::Paths::Root.new(Rails.root) + root.add('db/migrate') + end + + before do + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:[]).with('SKIP_POST_DEPLOYMENT_MIGRATIONS').and_return true + + # Our environment has already been loaded, so we need to pretent like post_migrations were not + allow(Rails.application.config.paths).to receive(:[]).and_call_original + allow(Rails.application.config.paths).to receive(:[]).with('db/migrate').and_return(migrations_paths) + allow(ActiveRecord::Migrator).to receive(:migrations_paths).and_return(migrations_paths) + end + + it 'adds post deployment migrations before schema load if the schema is not already loaded' do + allow(ActiveRecord::Base.connection).to receive(:tables).and_return([]) + expect(Gitlab::Database).to receive(:add_post_migrate_path_to_rails) + expect(Rake::Task['db:schema:load']).to receive(:invoke) + expect(Rake::Task['db:seed_fu']).to receive(:invoke) + expect(Rake::Task['db:migrate']).not_to receive(:invoke) + expect(Rails.application.config.paths).to receive(:[]).with('db/migrate') + expect { run_rake_task('gitlab:db:configure') }.not_to raise_error + + expect(migrations_paths.include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(true) + end + end end def run_rake_task(task_name) -- cgit v1.2.1 From 60747672bb5bca926c5a0323338bfc5dfd54b2bb Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Wed, 12 Sep 2018 15:55:26 -0700 Subject: Fix the schema load test And added changelog --- spec/tasks/gitlab/db_rake_spec.rb | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index 5e0300066d1..5818892d56a 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -63,31 +63,35 @@ describe 'gitlab:db namespace rake task' do end context 'SKIP_POST_DEPLOYMENT_MIGRATIONS environment variable set' do - let :migrations_paths do - root = Rails::Paths::Root.new(Rails.root) - root.add('db/migrate') - end + let(:rails_paths) { { 'db' => ['db'], 'db/migrate' => ['db/migrate'] } } before do allow(ENV).to receive(:[]).and_call_original allow(ENV).to receive(:[]).with('SKIP_POST_DEPLOYMENT_MIGRATIONS').and_return true - # Our environment has already been loaded, so we need to pretent like post_migrations were not - allow(Rails.application.config.paths).to receive(:[]).and_call_original - allow(Rails.application.config.paths).to receive(:[]).with('db/migrate').and_return(migrations_paths) - allow(ActiveRecord::Migrator).to receive(:migrations_paths).and_return(migrations_paths) + # Our environment has already been loaded, so we need to pretend like post_migrations were not + allow(Rails.application.config).to receive(:paths).and_return(rails_paths) + allow(ActiveRecord::Migrator).to receive(:migrations_paths).and_return(rails_paths['db/migrate'].dup) end it 'adds post deployment migrations before schema load if the schema is not already loaded' do allow(ActiveRecord::Base.connection).to receive(:tables).and_return([]) - expect(Gitlab::Database).to receive(:add_post_migrate_path_to_rails) + expect(Gitlab::Database).to receive(:add_post_migrate_path_to_rails).and_call_original expect(Rake::Task['db:schema:load']).to receive(:invoke) expect(Rake::Task['db:seed_fu']).to receive(:invoke) expect(Rake::Task['db:migrate']).not_to receive(:invoke) - expect(Rails.application.config.paths).to receive(:[]).with('db/migrate') expect { run_rake_task('gitlab:db:configure') }.not_to raise_error + expect(rails_paths['db/migrate'].include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(true) + end - expect(migrations_paths.include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(true) + it 'ignores post deployment migrations when schema has already been loaded' do + allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w[table1 table2]) + expect(Rake::Task['db:migrate']).to receive(:invoke) + expect(Gitlab::Database).not_to receive(:add_post_migrate_path_to_rails) + expect(Rake::Task['db:schema:load']).not_to receive(:invoke) + expect(Rake::Task['db:seed_fu']).not_to receive(:invoke) + expect { run_rake_task('gitlab:db:configure') }.not_to raise_error + expect(rails_paths['db/migrate'].include?(File.join(Rails.root, 'db', 'post_migrate'))).to be(false) end end end -- cgit v1.2.1 From 15d011d64d579616dd9df3e3e5082ed5b99bcdf8 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 25 Sep 2018 16:29:17 +0300 Subject: Geo: sync disabled wikis. Stage 2 We started syncing all the wiki regardless of the fact it's disabled or not. We couldn't do that in one stage because of needing of smoth update and deprecating things. This is the second stage that finally removes unused columns in the geo_node_status table. --- spec/tasks/gitlab/site_statistics_rake_spec.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/site_statistics_rake_spec.rb b/spec/tasks/gitlab/site_statistics_rake_spec.rb index 20f0df65e63..c43ce25a540 100644 --- a/spec/tasks/gitlab/site_statistics_rake_spec.rb +++ b/spec/tasks/gitlab/site_statistics_rake_spec.rb @@ -6,7 +6,7 @@ describe 'rake gitlab:refresh_site_statistics' do Rake.application.rake_require 'tasks/gitlab/site_statistics' create(:project) - SiteStatistic.fetch.update(repositories_count: 0, wikis_count: 0) + SiteStatistic.fetch.update(repositories_count: 0) end let(:task) { 'gitlab:refresh_site_statistics' } @@ -15,10 +15,9 @@ describe 'rake gitlab:refresh_site_statistics' do run_rake_task(task) expect(SiteStatistic.fetch.repositories_count).to eq(1) - expect(SiteStatistic.fetch.wikis_count).to eq(1) end it 'displays message listing counters' do - expect { run_rake_task(task) }.to output(/Updating Site Statistics counters:.* Repositories\.\.\. OK!.* Wikis\.\.\. OK!/m).to_stdout + expect { run_rake_task(task) }.to output(/Updating Site Statistics counters:.* Repositories\.\.\. OK!/m).to_stdout end end -- cgit v1.2.1 From 733ae9492129e835f183902a97ee0886e2dbdc9b Mon Sep 17 00:00:00 2001 From: George Tsiolis Date: Tue, 30 Oct 2018 12:53:01 +0200 Subject: Fix typos in comments and specs --- spec/tasks/gitlab/backup_rake_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 3ba6caf1337..8c4360d4cf0 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -251,7 +251,7 @@ describe 'gitlab:app namespace rake task' do allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) - # Avoid asking gitaly about the root ref (which will fail beacuse of the + # Avoid asking gitaly about the root ref (which will fail because of the # mocked storages) allow_any_instance_of(Repository).to receive(:empty?).and_return(false) end -- cgit v1.2.1 From 1c481b7aacdc7e90d0f349dc8e848adaf0813c65 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 5 Oct 2018 15:59:58 +0200 Subject: Enhance performance of counting local Uploads Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070 --- spec/tasks/gitlab/uploads/migrate_rake_spec.rb | 8 -------- 1 file changed, 8 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb index 6fcfae358ec..9588e8be5dc 100644 --- a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb +++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb @@ -38,14 +38,6 @@ describe 'gitlab:uploads:migrate rake tasks' do let!(:projects) { create_list(:project, 10, :with_avatar) } it_behaves_like 'enqueue jobs in batch', batch: 4 - - context 'Upload has store = nil' do - before do - Upload.where(model: projects).update_all(store: nil) - end - - it_behaves_like 'enqueue jobs in batch', batch: 4 - end end context "for Group" do -- cgit v1.2.1 From 6bb1a2ab23ba7729b1c9720296dfceb6079644c7 Mon Sep 17 00:00:00 2001 From: Heinrich Lee Yu Date: Fri, 23 Nov 2018 16:41:08 +0800 Subject: Update tests --- spec/tasks/cache/clear/redis_spec.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'spec/tasks') diff --git a/spec/tasks/cache/clear/redis_spec.rb b/spec/tasks/cache/clear/redis_spec.rb index cca2b864e9b..97c8c943f3a 100644 --- a/spec/tasks/cache/clear/redis_spec.rb +++ b/spec/tasks/cache/clear/redis_spec.rb @@ -6,7 +6,10 @@ describe 'clearing redis cache' do end describe 'clearing pipeline status cache' do - let(:pipeline_status) { create(:ci_pipeline).project.pipeline_status } + let(:pipeline_status) do + project = create(:project, :repository) + create(:ci_pipeline, project: project).project.pipeline_status + end before do allow(pipeline_status).to receive(:loaded).and_return(nil) -- cgit v1.2.1 From fe2e6c6dc046748f8760bfbb3c74185bdbf1359b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 23 Nov 2018 05:33:07 +0100 Subject: Remove Site Statistic This approach caused many different problems as we tightened the query execution timeout. --- spec/tasks/gitlab/site_statistics_rake_spec.rb | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 spec/tasks/gitlab/site_statistics_rake_spec.rb (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/site_statistics_rake_spec.rb b/spec/tasks/gitlab/site_statistics_rake_spec.rb deleted file mode 100644 index c43ce25a540..00000000000 --- a/spec/tasks/gitlab/site_statistics_rake_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true -require 'rake_helper' - -describe 'rake gitlab:refresh_site_statistics' do - before do - Rake.application.rake_require 'tasks/gitlab/site_statistics' - - create(:project) - SiteStatistic.fetch.update(repositories_count: 0) - end - - let(:task) { 'gitlab:refresh_site_statistics' } - - it 'recalculates existing counters' do - run_rake_task(task) - - expect(SiteStatistic.fetch.repositories_count).to eq(1) - end - - it 'displays message listing counters' do - expect { run_rake_task(task) }.to output(/Updating Site Statistics counters:.* Repositories\.\.\. OK!/m).to_stdout - end -end -- cgit v1.2.1 From 6855e6b5864abcf01689720424a4bea4c3b9fec2 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 27 Nov 2018 16:08:31 -0800 Subject: Extract system check rake task logic These changes make the code more reusable, testable, and most importantly, overrideable. --- spec/tasks/gitlab/check_rake_spec.rb | 108 +++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 29 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/check_rake_spec.rb b/spec/tasks/gitlab/check_rake_spec.rb index 4eda618b6d6..06525e3c771 100644 --- a/spec/tasks/gitlab/check_rake_spec.rb +++ b/spec/tasks/gitlab/check_rake_spec.rb @@ -1,51 +1,101 @@ require 'rake_helper' -describe 'gitlab:ldap:check rake task' do - include LdapHelpers - +describe 'check.rake' do before do Rake.application.rake_require 'tasks/gitlab/check' stub_warn_user_is_not_gitlab end - context 'when LDAP is not enabled' do - it 'does not attempt to bind or search for users' do - expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers) - expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open) - - run_rake_task('gitlab:ldap:check') + shared_examples_for 'system check rake task' do + it 'runs the check' do + expect do + subject + end.to output(/Checking #{name} ... Finished/).to_stdout end end - context 'when LDAP is enabled' do - let(:ldap) { double(:ldap) } - let(:adapter) { ldap_adapter('ldapmain', ldap) } + describe 'gitlab:check rake task' do + subject { run_rake_task('gitlab:check') } + let(:name) { 'GitLab subtasks' } - before do - allow(Gitlab::Auth::LDAP::Config) - .to receive_messages( - enabled?: true, - providers: ['ldapmain'] - ) - allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter) - allow(adapter).to receive(:users).and_return([]) - end + it_behaves_like 'system check rake task' + end + + describe 'gitlab:gitlab_shell:check rake task' do + subject { run_rake_task('gitlab:gitlab_shell:check') } + let(:name) { 'GitLab Shell' } + + it_behaves_like 'system check rake task' + end + + describe 'gitlab:gitaly:check rake task' do + subject { run_rake_task('gitlab:gitaly:check') } + let(:name) { 'Gitaly' } + + it_behaves_like 'system check rake task' + end + + describe 'gitlab:sidekiq:check rake task' do + subject { run_rake_task('gitlab:sidekiq:check') } + let(:name) { 'Sidekiq' } - it 'attempts to bind using credentials' do - stub_ldap_config(has_auth?: true) + it_behaves_like 'system check rake task' + end - expect(ldap).to receive(:bind) + describe 'gitlab:incoming_email:check rake task' do + subject { run_rake_task('gitlab:incoming_email:check') } + let(:name) { 'Incoming Email' } - run_rake_task('gitlab:ldap:check') + it_behaves_like 'system check rake task' + end + + describe 'gitlab:ldap:check rake task' do + include LdapHelpers + + subject { run_rake_task('gitlab:ldap:check') } + let(:name) { 'LDAP' } + + it_behaves_like 'system check rake task' + + context 'when LDAP is not enabled' do + it 'does not attempt to bind or search for users' do + expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers) + expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open) + + subject + end end - it 'searches for 100 LDAP users' do - stub_ldap_config(uid: 'uid') + context 'when LDAP is enabled' do + let(:ldap) { double(:ldap) } + let(:adapter) { ldap_adapter('ldapmain', ldap) } + + before do + allow(Gitlab::Auth::LDAP::Config) + .to receive_messages( + enabled?: true, + providers: ['ldapmain'] + ) + allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter) + allow(adapter).to receive(:users).and_return([]) + end + + it 'attempts to bind using credentials' do + stub_ldap_config(has_auth?: true) + + expect(ldap).to receive(:bind) + + subject + end + + it 'searches for 100 LDAP users' do + stub_ldap_config(uid: 'uid') - expect(adapter).to receive(:users).with('uid', '*', 100) + expect(adapter).to receive(:users).with('uid', '*', 100) - run_rake_task('gitlab:ldap:check') + subject + end end end end -- cgit v1.2.1 From 7329480412a4ea072b1af20d7fb35145e1a28db8 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 6 Dec 2018 18:30:20 +0000 Subject: Fix gitlab:web_hook tasks --- spec/tasks/gitlab/web_hook_rake_spec.rb | 92 +++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 spec/tasks/gitlab/web_hook_rake_spec.rb (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/web_hook_rake_spec.rb b/spec/tasks/gitlab/web_hook_rake_spec.rb new file mode 100644 index 00000000000..7bdf33ff6b0 --- /dev/null +++ b/spec/tasks/gitlab/web_hook_rake_spec.rb @@ -0,0 +1,92 @@ +require 'rake_helper' + +describe 'gitlab:web_hook namespace rake tasks' do + set(:group) { create(:group) } + + set(:project1) { create(:project, namespace: group) } + set(:project2) { create(:project, namespace: group) } + set(:other_group_project) { create(:project) } + + let(:url) { 'http://example.com' } + let(:hook_urls) { (project1.hooks + project2.hooks).map(&:url) } + let(:other_group_hook_urls) { other_group_project.hooks.map(&:url) } + + before do + Rake.application.rake_require 'tasks/gitlab/web_hook' + end + + describe 'gitlab:web_hook:add' do + it 'adds a web hook to all projects' do + stub_env('URL' => url) + run_rake_task('gitlab:web_hook:add') + + expect(hook_urls).to contain_exactly(url, url) + expect(other_group_hook_urls).to contain_exactly(url) + end + + it 'adds a web hook to projects in the specified namespace' do + stub_env('URL' => url, 'NAMESPACE' => group.full_path) + run_rake_task('gitlab:web_hook:add') + + expect(hook_urls).to contain_exactly(url, url) + expect(other_group_hook_urls).to be_empty + end + + it 'raises an error if an unknown namespace is specified' do + stub_env('URL' => url, 'NAMESPACE' => group.full_path) + + group.destroy + + expect { run_rake_task('gitlab:web_hook:add') }.to raise_error(SystemExit) + end + end + + describe 'gitlab:web_hook:rm' do + let!(:hook1) { create(:project_hook, project: project1, url: url) } + let!(:hook2) { create(:project_hook, project: project2, url: url) } + let!(:other_group_hook) { create(:project_hook, project: other_group_project, url: url) } + let!(:other_url_hook) { create(:project_hook, url: other_url, project: project1) } + + let(:other_url) { 'http://other.example.com' } + + it 'removes a web hook from all projects by URL' do + stub_env('URL' => url) + run_rake_task('gitlab:web_hook:rm') + + expect(hook_urls).to contain_exactly(other_url) + expect(other_group_hook_urls).to be_empty + end + + it 'removes a web hook from projects in the specified namespace by URL' do + stub_env('NAMESPACE' => group.full_path, 'URL' => url) + run_rake_task('gitlab:web_hook:rm') + + expect(hook_urls).to contain_exactly(other_url) + expect(other_group_hook_urls).to contain_exactly(url) + end + + it 'raises an error if an unknown namespace is specified' do + stub_env('URL' => url, 'NAMESPACE' => group.full_path) + + group.destroy + + expect { run_rake_task('gitlab:web_hook:rm') }.to raise_error(SystemExit) + end + end + + describe 'gitlab:web_hook:list' do + let!(:hook1) { create(:project_hook, project: project1) } + let!(:hook2) { create(:project_hook, project: project2) } + let!(:other_group_hook) { create(:project_hook, project: other_group_project) } + + it 'lists all web hooks' do + expect { run_rake_task('gitlab:web_hook:list') }.to output(/3 webhooks found/).to_stdout + end + + it 'lists web hooks in a particular namespace' do + stub_env('NAMESPACE', group.full_path) + + expect { run_rake_task('gitlab:web_hook:list') }.to output(/2 webhooks found/).to_stdout + end + end +end -- cgit v1.2.1 From 89a407dc3bea38b60e06eb825991cbea0c87b85a Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Mon, 10 Dec 2018 09:31:28 +0100 Subject: Restore Object Pools when restoring an object pool Pool repositories are persisted in the database, and when the DB is restored, the data need to be restored on disk. This is done by resetting the state machine and rescheduling the object pool creation. This is not an exact replica of the state like at the time of the creation of the backup. However, the data is consistent again. Dumping isn't required as internally GitLab uses git bundles which bundle all refs and include all objects in the bundle that they require, reduplicating as more repositories get backed up. This does require more data to be stored. Fixes https://gitlab.com/gitlab-org/gitaly/issues/1355 --- spec/tasks/gitlab/backup_rake_spec.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 8c4360d4cf0..3b8f7f5fe7d 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -74,6 +74,7 @@ describe 'gitlab:app namespace rake task' do it 'invokes restoration on match' do allow(YAML).to receive(:load_file) .and_return({ gitlab_version: gitlab_version }) + expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke) expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke) expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke) -- cgit v1.2.1 From 9f80f0405968dff3ec65b59fcc291c0ee87cdf3a Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 19 Dec 2018 21:10:00 +0000 Subject: Prevent admins from attempting hashed storage migration on read only DB --- spec/tasks/gitlab/storage_rake_spec.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index 233076ad6fa..be902d7c679 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -46,6 +46,16 @@ describe 'rake gitlab:storage:*' do describe 'gitlab:storage:migrate_to_hashed' do let(:task) { 'gitlab:storage:migrate_to_hashed' } + context 'read-only database' do + it 'does nothing' do + expect(Gitlab::Database).to receive(:read_only?).and_return(true) + + expect(Project).not_to receive(:with_unmigrated_storage) + + expect { run_rake_task(task) }.to output(/This task requires database write access. Exiting./).to_stderr + end + end + context '0 legacy projects' do it 'does nothing' do expect(StorageMigratorWorker).not_to receive(:perform_async) @@ -92,7 +102,7 @@ describe 'rake gitlab:storage:*' do stub_env('ID_FROM', 99999) stub_env('ID_TO', 99999) - expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stdout + expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stderr end it 'displays a message when project exists but its already migrated' do @@ -100,7 +110,7 @@ describe 'rake gitlab:storage:*' do stub_env('ID_FROM', project.id) stub_env('ID_TO', project.id) - expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stdout + expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stderr end it 'enqueues migration when project can be found' do -- cgit v1.2.1 From c2c34eba62f26bed8cad8b07934e14b4519eef7c Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Sat, 22 Dec 2018 14:54:33 +0100 Subject: Prepare rake task for storage rollback We are keeping compatibility with existing scheduled jobs. --- spec/tasks/gitlab/storage_rake_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index be902d7c679..3b8d5ad7015 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -74,7 +74,7 @@ describe 'rake gitlab:storage:*' do it 'enqueues one StorageMigratorWorker per project' do projects.each do |project| - expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id) + expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id, :migrate) end run_rake_task(task) @@ -89,7 +89,7 @@ describe 'rake gitlab:storage:*' do it 'enqueues one StorageMigratorWorker per 2 projects' do projects.map(&:id).sort.each_slice(2) do |first, last| last ||= first - expect(StorageMigratorWorker).to receive(:perform_async).with(first, last) + expect(StorageMigratorWorker).to receive(:perform_async).with(first, last, :migrate) end run_rake_task(task) -- cgit v1.2.1 From 7bc16889df458865ffbbb7bef8087c04a5768a1d Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 17 Jan 2019 02:53:50 +0100 Subject: Refactor Storage Migration Specs were reviewed and improved to better cover the current behavior. There was some standardization done as well to facilitate the implementation of the rollback functionality. StorageMigratorWorker was extracted to HashedStorage namespace were RollbackerWorker will live one as well. --- spec/tasks/gitlab/storage_rake_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'spec/tasks') diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index 3b8d5ad7015..6b50670c3c0 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -58,7 +58,7 @@ describe 'rake gitlab:storage:*' do context '0 legacy projects' do it 'does nothing' do - expect(StorageMigratorWorker).not_to receive(:perform_async) + expect(::HashedStorage::MigratorWorker).not_to receive(:perform_async) run_rake_task(task) end @@ -72,9 +72,9 @@ describe 'rake gitlab:storage:*' do stub_env('BATCH' => 1) end - it 'enqueues one StorageMigratorWorker per project' do + it 'enqueues one HashedStorage::MigratorWorker per project' do projects.each do |project| - expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id, :migrate) + expect(::HashedStorage::MigratorWorker).to receive(:perform_async).with(project.id, project.id) end run_rake_task(task) @@ -86,10 +86,10 @@ describe 'rake gitlab:storage:*' do stub_env('BATCH' => 2) end - it 'enqueues one StorageMigratorWorker per 2 projects' do + it 'enqueues one HashedStorage::MigratorWorker per 2 projects' do projects.map(&:id).sort.each_slice(2) do |first, last| last ||= first - expect(StorageMigratorWorker).to receive(:perform_async).with(first, last, :migrate) + expect(::HashedStorage::MigratorWorker).to receive(:perform_async).with(first, last) end run_rake_task(task) -- cgit v1.2.1