Add local project uploads cleanup task

1 files changed, 316 insertions, 1 deletions

diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb
index 2bf873c923f..ba08ece1b4b 100644
--- a/spec/tasks/gitlab/cleanup_rake_spec.rb
+++ b/spec/tasks/gitlab/cleanup_rake_spec.rb
@@ -5,7 +5,7 @@ describe 'gitlab:cleanup rake tasks' do
     Rake.application.rake_require 'tasks/gitlab/cleanup'
   end
 
-  describe 'cleanup' do
+  describe 'cleanup namespaces and repos' do
     let(:storages) do
       {
         'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
@@ -67,4 +67,319 @@ describe 'gitlab:cleanup rake tasks' do
       end
     end
   end
+
+  describe 'cleanup:project_uploads' do
+    context 'orphaned project upload file' do
+      context 'when an upload record matching the secret and filename is found' do
+        context 'when the project is still in legacy storage' do
+          let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) }
+          let!(:correct_path) { orphaned.absolute_path }
+          let!(:other_project) { create(:project, :legacy_storage) }
+          let!(:orphaned_path) { correct_path.sub(/#{orphaned.model.full_path}/, other_project.full_path) }
+
+          before do
+            FileUtils.mkdir_p(File.dirname(orphaned_path))
+            FileUtils.mv(correct_path, orphaned_path)
+          end
+
+          it 'moves the file to its proper location' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}")
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(correct_path)).to be_falsey
+
+            stub_env('DRY_RUN', 'false')
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_falsey
+            expect(File.exist?(correct_path)).to be_truthy
+          end
+
+          it 'a dry run does not move the file' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}")
+            expect(Rails.logger).to receive(:info)
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(correct_path)).to be_falsey
+
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(correct_path)).to be_falsey
+          end
+
+          context 'when the project record is missing (Upload#absolute_path raises error)' do
+            let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', other_project.full_path, orphaned.path) }
+
+            before do
+              orphaned.model.delete
+            end
+
+            it 'moves the file to lost and found' do
+              expect(Rails.logger).to receive(:info).twice
+              expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
+
+              expect(File.exist?(orphaned_path)).to be_truthy
+              expect(File.exist?(lost_and_found_path)).to be_falsey
+
+              stub_env('DRY_RUN', 'false')
+              run_rake_task('gitlab:cleanup:project_uploads')
+
+              expect(File.exist?(orphaned_path)).to be_falsey
+              expect(File.exist?(lost_and_found_path)).to be_truthy
+            end
+
+            it 'a dry run does not move the file' do
+              expect(Rails.logger).to receive(:info).twice
+              expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
+              expect(Rails.logger).to receive(:info)
+
+              expect(File.exist?(orphaned_path)).to be_truthy
+              expect(File.exist?(lost_and_found_path)).to be_falsey
+
+              run_rake_task('gitlab:cleanup:project_uploads')
+
+              expect(File.exist?(orphaned_path)).to be_truthy
+              expect(File.exist?(lost_and_found_path)).to be_falsey
+            end
+          end
+        end
+
+        context 'when the project was moved to hashed storage' do
+          let!(:orphaned) { create(:upload, :issuable_upload, :with_file) }
+          let!(:correct_path) { orphaned.absolute_path }
+          let!(:orphaned_path) { File.join(FileUploader.root, 'foo', 'bar', orphaned.path) }
+
+          before do
+            FileUtils.mkdir_p(File.dirname(orphaned_path))
+            FileUtils.mv(correct_path, orphaned_path)
+          end
+
+          it 'moves the file to its proper location' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}")
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(correct_path)).to be_falsey
+
+            stub_env('DRY_RUN', 'false')
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_falsey
+            expect(File.exist?(correct_path)).to be_truthy
+          end
+
+          it 'a dry run does not move the file' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}")
+            expect(Rails.logger).to receive(:info)
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(correct_path)).to be_falsey
+
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(correct_path)).to be_falsey
+          end
+        end
+      end
+
+      context 'when a matching upload record can not be found' do
+        context 'when the file path fits the known pattern' do
+          let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) }
+          let!(:orphaned_path) { orphaned.absolute_path }
+          let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', orphaned.model.full_path, orphaned.path) }
+
+          before do
+            orphaned.delete
+          end
+
+          it 'moves the file to lost and found' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(lost_and_found_path)).to be_falsey
+
+            stub_env('DRY_RUN', 'false')
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_falsey
+            expect(File.exist?(lost_and_found_path)).to be_truthy
+          end
+
+          it 'a dry run does not move the file' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
+            expect(Rails.logger).to receive(:info)
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(lost_and_found_path)).to be_falsey
+
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(lost_and_found_path)).to be_falsey
+          end
+        end
+
+        context 'when the file path does not fit the known pattern' do
+          let!(:invalid_path) { File.join('group', 'file.jpg') }
+          let!(:orphaned_path) { File.join(FileUploader.root, invalid_path) }
+          let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', invalid_path) }
+
+          before do
+            FileUtils.mkdir_p(File.dirname(orphaned_path))
+            FileUtils.touch(orphaned_path)
+          end
+
+          after do
+            File.delete(orphaned_path) if File.exist?(orphaned_path)
+          end
+
+          it 'moves the file to lost and found' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(lost_and_found_path)).to be_falsey
+
+            stub_env('DRY_RUN', 'false')
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_falsey
+            expect(File.exist?(lost_and_found_path)).to be_truthy
+          end
+
+          it 'a dry run does not move the file' do
+            expect(Rails.logger).to receive(:info).twice
+            expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
+            expect(Rails.logger).to receive(:info)
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(lost_and_found_path)).to be_falsey
+
+            run_rake_task('gitlab:cleanup:project_uploads')
+
+            expect(File.exist?(orphaned_path)).to be_truthy
+            expect(File.exist?(lost_and_found_path)).to be_falsey
+          end
+        end
+      end
+    end
+
+    context 'non-orphaned project upload file' do
+      it 'does not move the file' do
+        tracked = create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage))
+        tracked_path = tracked.absolute_path
+
+        expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
+        expect(File.exist?(tracked_path)).to be_truthy
+
+        stub_env('DRY_RUN', 'false')
+        run_rake_task('gitlab:cleanup:project_uploads')
+
+        expect(File.exist?(tracked_path)).to be_truthy
+      end
+    end
+
+    context 'ignorable cases' do
+      shared_examples_for 'does not move anything' do
+        it 'does not move even an orphan file' do
+          orphaned = create(:upload, :issuable_upload, :with_file, model: project)
+          orphaned_path = orphaned.absolute_path
+          orphaned.delete
+
+          expect(File.exist?(orphaned_path)).to be_truthy
+
+          run_rake_task('gitlab:cleanup:project_uploads')
+
+          expect(File.exist?(orphaned_path)).to be_truthy
+        end
+      end
+
+      # Because we aren't concerned about these, and can save a lot of
+      # processing time by ignoring them. If we wish to cleanup hashed storage
+      # directories, it should simply require removing this test and modifying
+      # the find command.
+      context 'when the file is already in hashed storage' do
+        let(:project) { create(:project) }
+
+        before do
+          stub_env('DRY_RUN', 'false')
+          expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
+        end
+
+        it_behaves_like 'does not move anything'
+      end
+
+      context 'when DRY_RUN env var is unset' do
+        let(:project) { create(:project, :legacy_storage) }
+
+        it_behaves_like 'does not move anything'
+      end
+
+      context 'when DRY_RUN env var is true' do
+        let(:project) { create(:project, :legacy_storage) }
+
+        before do
+          stub_env('DRY_RUN', 'true')
+        end
+
+        it_behaves_like 'does not move anything'
+      end
+
+      context 'when DRY_RUN env var is foo' do
+        let(:project) { create(:project, :legacy_storage) }
+
+        before do
+          stub_env('DRY_RUN', 'foo')
+        end
+
+        it_behaves_like 'does not move anything'
+      end
+
+      it 'does not move any non-project (FileUploader) uploads' do
+        stub_env('DRY_RUN', 'false')
+
+        paths = []
+        orphaned1 = create(:upload, :personal_snippet_upload, :with_file)
+        orphaned2 = create(:upload, :namespace_upload, :with_file)
+        orphaned3 = create(:upload, :attachment_upload, :with_file)
+        paths << orphaned1.absolute_path
+        paths << orphaned2.absolute_path
+        paths << orphaned3.absolute_path
+        Upload.delete_all
+
+        expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
+        paths.each do |path|
+          expect(File.exist?(path)).to be_truthy
+        end
+
+        run_rake_task('gitlab:cleanup:project_uploads')
+
+        paths.each do |path|
+          expect(File.exist?(path)).to be_truthy
+        end
+      end
+
+      it 'does not move any uploads in tmp (which would interfere with ongoing upload activity)' do
+        stub_env('DRY_RUN', 'false')
+
+        path = File.join(FileUploader.root, 'tmp', 'foo.jpg')
+        FileUtils.mkdir_p(File.dirname(path))
+        FileUtils.touch(path)
+
+        expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
+        expect(File.exist?(path)).to be_truthy
+
+        run_rake_task('gitlab:cleanup:project_uploads')
+
+        expect(File.exist?(path)).to be_truthy
+      end
+    end
+  end
 end