[ci skip] Merge branch 'master' into 44427-state-management-with-vuex

* master: (544 commits)
  Bulk deleting refs is handled by Gitaly by default
  Allow assigning and filtering issuables by ancestor group labels
  Fix links to subdirectories of a directory with a plus character in its path
  Add banzai filter to detect commit message trailers and properly link the users
  Render MR commit SHA instead "diffs" when viable
  Fix a transient failure by removing unneeded expectations
  Revert changelog entry for removed feature
  Revert "Allow CI/CD Jobs being grouped on version strings"
  Add support for Sidekiq JSON logging
  Resolve "Protected branches count is wrong when a wildcard includes several protected branches"
  Remove unused form for admin application settings
  Use standard codequality job
  Resolve "Allow the configuration of a project's merge method via the API"
  Move the rest of application settings to expandable blocks
  [Rails5] Rename `sort` methods to `sort_by_attribute`
  Add better LDAP connection handling
  Updated components to PascalCase
  Handle invalid params when trying update_username
  Move network related app settings to expandable blocks
  [Rails5] Update Gemfile.rails5.lock [ci skip]
  ...

8 files changed, 326 insertions, 13 deletions

diff --git a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb
new file mode 100644
index 00000000000..8544fb62b5a
--- /dev/null
+++ b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb
@@ -0,0 +1,118 @@
+require 'rake_helper'
+
+describe 'gitlab:artifacts namespace rake task' do
+  before(:context) do
+    Rake.application.rake_require 'tasks/gitlab/artifacts/migrate'
+  end
+
+  let(:object_storage_enabled) { false }
+
+  before do
+    stub_artifacts_object_storage(enabled: object_storage_enabled)
+  end
+
+  subject { run_rake_task('gitlab:artifacts:migrate') }
+
+  context 'legacy artifacts' do
+    describe 'migrate' do
+      let!(:build) { create(:ci_build, :legacy_artifacts, artifacts_file_store: store, artifacts_metadata_store: store) }
+
+      context 'when local storage is used' do
+        let(:store) { ObjectStorage::Store::LOCAL }
+
+        context 'and job does not have file store defined' do
+          let(:object_storage_enabled) { true }
+          let(:store) { nil }
+
+          it "migrates file to remote storage" do
+            subject
+
+            expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+            expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+          end
+        end
+
+        context 'and remote storage is defined' do
+          let(:object_storage_enabled) { true }
+
+          it "migrates file to remote storage" do
+            subject
+
+            expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+            expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+          end
+        end
+
+        context 'and remote storage is not defined' do
+          it "fails to migrate to remote storage" do
+            subject
+
+            expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::LOCAL)
+            expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::LOCAL)
+          end
+        end
+      end
+
+      context 'when remote storage is used' do
+        let(:object_storage_enabled) { true }
+
+        let(:store) { ObjectStorage::Store::REMOTE }
+
+        it "file stays on remote storage" do
+          subject
+
+          expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE)
+          expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+    end
+  end
+
+  context 'job artifacts' do
+    let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) }
+
+    context 'when local storage is used' do
+      let(:store) { ObjectStorage::Store::LOCAL }
+
+      context 'and job does not have file store defined' do
+        let(:object_storage_enabled) { true }
+        let(:store) { nil }
+
+        it "migrates file to remote storage" do
+          subject
+
+          expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is defined' do
+        let(:object_storage_enabled) { true }
+
+        it "migrates file to remote storage" do
+          subject
+
+          expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+        end
+      end
+
+      context 'and remote storage is not defined' do
+        it "fails to migrate to remote storage" do
+          subject
+
+          expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL)
+        end
+      end
+    end
+
+    context 'when remote storage is used' do
+      let(:object_storage_enabled) { true }
+      let(:store) { ObjectStorage::Store::REMOTE }
+
+      it "file stays on remote storage" do
+        subject
+
+        expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE)
+      end
+    end
+  end
+end
diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 168facd51a6..0d24782f317 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -195,14 +195,23 @@ describe 'gitlab:app namespace rake task' do
     end
 
     context 'multiple repository storages' do
-      let(:gitaly_address) { Gitlab.config.repositories.storages.default.gitaly_address }
+      let(:storage_default) do
+        Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
+      end
+      let(:test_second_storage) do
+        Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/custom_storage'))
+      end
       let(:storages) do
         {
-          'default' => { 'path' => Settings.absolute('tmp/tests/default_storage'), 'gitaly_address' => gitaly_address  },
-          'test_second_storage' => { 'path' => Settings.absolute('tmp/tests/custom_storage'), 'gitaly_address' => gitaly_address }
+          'default' => storage_default,
+          'test_second_storage' => test_second_storage
         }
       end
 
+      before(:all) do
+        @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
+      end
+
       before do
         # We only need a backup of the repositories for this test
         stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb
index 9e746ceddd6..2bf873c923f 100644
--- a/spec/tasks/gitlab/cleanup_rake_spec.rb
+++ b/spec/tasks/gitlab/cleanup_rake_spec.rb
@@ -6,13 +6,16 @@ describe 'gitlab:cleanup rake tasks' do
   end
 
   describe 'cleanup' do
-    let(:gitaly_address) { Gitlab.config.repositories.storages.default.gitaly_address }
     let(:storages) do
       {
-        'default' => { 'path' => Settings.absolute('tmp/tests/default_storage'), 'gitaly_address' => gitaly_address  }
+        'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
       }
     end
 
+    before(:all) do
+      @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
+    end
+
     before do
       FileUtils.mkdir(Settings.absolute('tmp/tests/default_storage'))
       allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb
index 9aebf7b0b4a..1efaecc63a5 100644
--- a/spec/tasks/gitlab/git_rake_spec.rb
+++ b/spec/tasks/gitlab/git_rake_spec.rb
@@ -1,10 +1,13 @@
 require 'rake_helper'
 
 describe 'gitlab:git rake tasks' do
+  before(:all) do
+    @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
+  end
+
   before do
     Rake.application.rake_require 'tasks/gitlab/git'
-
-    storages = { 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage') } }
+    storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) }
 
     FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git'))
     allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb
index 1f4053ff9ad..1e507c0236e 100644
--- a/spec/tasks/gitlab/gitaly_rake_spec.rb
+++ b/spec/tasks/gitlab/gitaly_rake_spec.rb
@@ -99,14 +99,14 @@ describe 'gitlab:gitaly namespace rake task' do
   describe 'storage_config' do
     it 'prints storage configuration in a TOML format' do
       config = {
-        'default' => {
+        'default' => Gitlab::GitalyClient::StorageSettings.new(
           'path' => '/path/to/default',
           'gitaly_address' => 'unix:/path/to/my.socket'
-        },
-        'nfs_01' => {
+        ),
+        'nfs_01' => Gitlab::GitalyClient::StorageSettings.new(
           'path' => '/path/to/nfs_01',
           'gitaly_address' => 'unix:/path/to/my.socket'
-        }
+        )
       }
       allow(Gitlab.config.repositories).to receive(:storages).and_return(config)
       allow(Rails.env).to receive(:test?).and_return(false)
@@ -134,7 +134,7 @@ describe 'gitlab:gitaly namespace rake task' do
 
       parsed_output = TomlRB.parse(expected_output)
       config.each do |name, params|
-        expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params['path'] })
+        expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path })
       end
     end
   end
diff --git a/spec/tasks/gitlab/lfs/migrate_rake_spec.rb b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
new file mode 100644
index 00000000000..66d1a192a96
--- /dev/null
+++ b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
@@ -0,0 +1,37 @@
+require 'rake_helper'
+
+describe 'gitlab:lfs namespace rake task' do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/lfs/migrate'
+  end
+
+  describe 'migrate' do
+    let(:local) { ObjectStorage::Store::LOCAL }
+    let(:remote) { ObjectStorage::Store::REMOTE }
+    let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) }
+
+    def lfs_migrate
+      run_rake_task('gitlab:lfs:migrate')
+    end
+
+    context 'object storage disabled' do
+      before do
+        stub_lfs_object_storage(enabled: false)
+      end
+
+      it "doesn't migrate files" do
+        expect { lfs_migrate }.not_to change { lfs_object.reload.file_store }
+      end
+    end
+
+    context 'object storage enabled' do
+      before do
+        stub_lfs_object_storage
+      end
+
+      it 'migrates local file to object storage' do
+        expect { lfs_migrate }.to change { lfs_object.reload.file_store }.from(local).to(remote)
+      end
+    end
+  end
+end
diff --git a/spec/tasks/gitlab/shell_rake_spec.rb b/spec/tasks/gitlab/shell_rake_spec.rb
index 65155cb044d..4a756c5742d 100644
--- a/spec/tasks/gitlab/shell_rake_spec.rb
+++ b/spec/tasks/gitlab/shell_rake_spec.rb
@@ -11,7 +11,7 @@ describe 'gitlab:shell rake tasks' do
     it 'invokes create_hooks task' do
       expect(Rake::Task['gitlab:shell:create_hooks']).to receive(:invoke)
 
-      storages = Gitlab.config.repositories.storages.values.map { |rs| rs['path'] }
+      storages = Gitlab.config.repositories.storages.values.map(&:legacy_disk_path)
       expect(Kernel).to receive(:system).with('bin/install', *storages).and_call_original
       expect(Kernel).to receive(:system).with('bin/compile').and_call_original
 
diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
new file mode 100644
index 00000000000..6fcfae358ec
--- /dev/null
+++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb
@@ -0,0 +1,143 @@
+require 'rake_helper'
+
+describe 'gitlab:uploads:migrate rake tasks' do
+  let(:model_class) { nil }
+  let(:uploader_class) { nil }
+  let(:mounted_as) { nil }
+  let(:batch_size) { 3 }
+
+  before do
+    stub_env('BATCH', batch_size.to_s)
+    stub_uploads_object_storage(uploader_class)
+    Rake.application.rake_require 'tasks/gitlab/uploads/migrate'
+
+    allow(ObjectStorage::MigrateUploadsWorker).to receive(:perform_async)
+  end
+
+  def run
+    args = [uploader_class.to_s, model_class.to_s, mounted_as].compact
+    run_rake_task("gitlab:uploads:migrate", *args)
+  end
+
+  shared_examples 'enqueue jobs in batch' do |batch:|
+    it do
+      expect(ObjectStorage::MigrateUploadsWorker)
+        .to receive(:perform_async).exactly(batch).times
+              .and_return("A fake job.")
+
+      run
+    end
+  end
+
+  context "for AvatarUploader" do
+    let(:uploader_class) { AvatarUploader }
+    let(:mounted_as) { :avatar }
+
+    context "for Project" do
+      let(:model_class) { Project }
+      let!(:projects) { create_list(:project, 10, :with_avatar) }
+
+      it_behaves_like 'enqueue jobs in batch', batch: 4
+
+      context 'Upload has store = nil' do
+        before do
+          Upload.where(model: projects).update_all(store: nil)
+        end
+
+        it_behaves_like 'enqueue jobs in batch', batch: 4
+      end
+    end
+
+    context "for Group" do
+      let(:model_class) { Group }
+
+      before do
+        create_list(:group, 10, :with_avatar)
+      end
+
+      it_behaves_like 'enqueue jobs in batch', batch: 4
+    end
+
+    context "for User" do
+      let(:model_class) { User }
+
+      before do
+        create_list(:user, 10, :with_avatar)
+      end
+
+      it_behaves_like 'enqueue jobs in batch', batch: 4
+    end
+  end
+
+  context "for AttachmentUploader" do
+    let(:uploader_class) { AttachmentUploader }
+
+    context "for Note" do
+      let(:model_class) { Note }
+      let(:mounted_as) { :attachment }
+
+      before do
+        create_list(:note, 10, :with_attachment)
+      end
+
+      it_behaves_like 'enqueue jobs in batch', batch: 4
+    end
+
+    context "for Appearance" do
+      let(:model_class) { Appearance }
+      let(:mounted_as) { :logo }
+
+      before do
+        create(:appearance, :with_logos)
+      end
+
+      %i(logo header_logo).each do |mount|
+        it_behaves_like 'enqueue jobs in batch', batch: 1 do
+          let(:mounted_as) { mount }
+        end
+      end
+    end
+  end
+
+  context "for FileUploader" do
+    let(:uploader_class) { FileUploader }
+    let(:model_class) { Project }
+
+    before do
+      create_list(:project, 10) do |model|
+        uploader_class.new(model)
+          .store!(fixture_file_upload('spec/fixtures/doc_sample.txt'))
+      end
+    end
+
+    it_behaves_like 'enqueue jobs in batch', batch: 4
+  end
+
+  context "for PersonalFileUploader" do
+    let(:uploader_class) { PersonalFileUploader }
+    let(:model_class) { PersonalSnippet }
+
+    before do
+      create_list(:personal_snippet, 10) do |model|
+        uploader_class.new(model)
+          .store!(fixture_file_upload('spec/fixtures/doc_sample.txt'))
+      end
+    end
+
+    it_behaves_like 'enqueue jobs in batch', batch: 4
+  end
+
+  context "for NamespaceFileUploader" do
+    let(:uploader_class) { NamespaceFileUploader }
+    let(:model_class) { Snippet }
+
+    before do
+      create_list(:snippet, 10) do |model|
+        uploader_class.new(model)
+          .store!(fixture_file_upload('spec/fixtures/doc_sample.txt'))
+      end
+    end
+
+    it_behaves_like 'enqueue jobs in batch', batch: 4
+  end
+end