Merge branch 'master' into backstage/gb/migrate-stages-statuses

* master: (319 commits)
  remove redundant changelog entries
  Merge branch '24570-use-re2-for-user-supplied-regexp-9-3' into 'security-9-3'
  Merge branch '33303-404-for-unauthorized-project' into 'security-9-3'
  Merge branch '33359-pers-snippet-files-location' into 'security-9-3'
  Merge branch 'bvl-remove-appearance-symlink' into 'security-9-3'
  Hide description about protected branches to non-member
  Update CHANGELOG.md for 9.0.11
  Update CHANGELOG.md for 9.1.8
  Update CHANGELOG.md for 8.17.7
  Update CHANGELOG.md for 9.2.8
  Update CHANGELOG.md for 9.3.8
  Respect blockquote line breaks in markdown
  35209 Add wip message to new navigation preference section
  Add github imported projects count to usage data
  Add versions to Prometheus metrics doc
  Add Bulgarian translations of Pipeline Schedules
  Add Esperanto translations of Pipeline Schedules
  Add Traditional Chinese in HongKong translations of Pipeline Schedules
  Add Simplified Chinese translations of Pipeline Schedules
  Resolve "Clarify k8s service keys"
  ...

Conflicts:
	db/schema.rb

7 files changed, 366 insertions, 4 deletions

diff --git a/spec/migrations/add_foreign_key_to_merge_requests_spec.rb b/spec/migrations/add_foreign_key_to_merge_requests_spec.rb
new file mode 100644
index 00000000000..d9ad9a585f0
--- /dev/null
+++ b/spec/migrations/add_foreign_key_to_merge_requests_spec.rb
@@ -0,0 +1,39 @@
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20170713104829_add_foreign_key_to_merge_requests.rb')
+
+describe AddForeignKeyToMergeRequests, :migration do
+  let(:projects) { table(:projects) }
+  let(:merge_requests) { table(:merge_requests) }
+  let(:pipelines) { table(:ci_pipelines) }
+
+  before do
+    projects.create!(name: 'gitlab', path: 'gitlab-org/gitlab-ce')
+    pipelines.create!(project_id: projects.first.id,
+                      ref: 'some-branch',
+                      sha: 'abc12345')
+
+    # merge request without a pipeline
+    create_merge_request(head_pipeline_id: nil)
+
+    # merge request with non-existent pipeline
+    create_merge_request(head_pipeline_id: 1234)
+
+    # merge reqeust with existing pipeline assigned
+    create_merge_request(head_pipeline_id: pipelines.first.id)
+  end
+
+  it 'correctly adds a foreign key to head_pipeline_id' do
+    migrate!
+
+    expect(merge_requests.first.head_pipeline_id).to be_nil
+    expect(merge_requests.second.head_pipeline_id).to be_nil
+    expect(merge_requests.third.head_pipeline_id).to eq pipelines.first.id
+  end
+
+  def create_merge_request(**opts)
+    merge_requests.create!(source_project_id: projects.first.id,
+                           target_project_id: projects.first.id,
+                           source_branch: 'some-branch',
+                           target_branch: 'master', **opts)
+  end
+end
diff --git a/spec/migrations/clean_appearance_symlinks_spec.rb b/spec/migrations/clean_appearance_symlinks_spec.rb
new file mode 100644
index 00000000000..9225dc0d894
--- /dev/null
+++ b/spec/migrations/clean_appearance_symlinks_spec.rb
@@ -0,0 +1,46 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170613111224_clean_appearance_symlinks.rb')
+
+describe CleanAppearanceSymlinks do
+  let(:migration) { described_class.new }
+  let(:test_dir) { File.join(Rails.root, "tmp", "tests", "clean_appearance_test") }
+  let(:uploads_dir) { File.join(test_dir, "public", "uploads") }
+  let(:new_uploads_dir) { File.join(uploads_dir, "system") }
+  let(:original_path) { File.join(new_uploads_dir, 'appearance') }
+  let(:symlink_path) { File.join(uploads_dir, 'appearance') }
+
+  before do
+    FileUtils.remove_dir(test_dir) if File.directory?(test_dir)
+    FileUtils.mkdir_p(uploads_dir)
+    allow(migration).to receive(:base_directory).and_return(test_dir)
+    allow(migration).to receive(:say)
+  end
+
+  describe "#up" do
+    before do
+      FileUtils.mkdir_p(original_path)
+      FileUtils.ln_s(original_path, symlink_path)
+    end
+
+    it 'removes the symlink' do
+      migration.up
+
+      expect(File.symlink?(symlink_path)).to be(false)
+    end
+  end
+
+  describe '#down' do
+    before do
+      FileUtils.mkdir_p(File.join(original_path))
+      FileUtils.touch(File.join(original_path, 'dummy.file'))
+    end
+
+    it 'creates a symlink' do
+      expected_path = File.join(symlink_path, "dummy.file")
+      migration.down
+
+      expect(File.exist?(expected_path)).to be(true)
+      expect(File.symlink?(symlink_path)).to be(true)
+    end
+  end
+end
diff --git a/spec/migrations/cleanup_move_system_upload_folder_symlink_spec.rb b/spec/migrations/cleanup_move_system_upload_folder_symlink_spec.rb
new file mode 100644
index 00000000000..3a9fa8c7113
--- /dev/null
+++ b/spec/migrations/cleanup_move_system_upload_folder_symlink_spec.rb
@@ -0,0 +1,35 @@
+require 'spec_helper'
+require Rails.root.join("db", "post_migrate", "20170717111152_cleanup_move_system_upload_folder_symlink.rb")
+
+describe CleanupMoveSystemUploadFolderSymlink do
+  let(:migration) { described_class.new }
+  let(:test_base) { File.join(Rails.root, 'tmp', 'tests', 'move-system-upload-folder') }
+  let(:test_folder) { File.join(test_base, '-', 'system') }
+
+  before do
+    allow(migration).to receive(:base_directory).and_return(test_base)
+    FileUtils.rm_rf(test_base)
+    FileUtils.mkdir_p(test_folder)
+    allow(migration).to receive(:say)
+  end
+
+  describe '#up' do
+    before do
+      FileUtils.ln_s(test_folder, File.join(test_base, 'system'))
+    end
+
+    it 'removes the symlink' do
+      migration.up
+
+      expect(File.exist?(File.join(test_base, 'system'))).to be_falsey
+    end
+  end
+
+  describe '#down' do
+    it 'creates the symlink' do
+      migration.down
+
+      expect(File.symlink?(File.join(test_base, 'system'))).to be_truthy
+    end
+  end
+end
diff --git a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb
index 4223d2337a8..5b633dd349b 100644
--- a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb
+++ b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb
@@ -54,7 +54,7 @@ describe MigrateProcessCommitWorkerJobs do
     end
   end
 
-  describe '#up', :redis do
+  describe '#up', :clean_gitlab_redis_shared_state do
     let(:migration) { described_class.new }
 
     def job_count
@@ -172,7 +172,7 @@ describe MigrateProcessCommitWorkerJobs do
     end
   end
 
-  describe '#down', :redis do
+  describe '#down', :clean_gitlab_redis_shared_state do
     let(:migration) { described_class.new }
 
     def job_count
diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb
index e3b42b5eac8..063829be546 100644
--- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb
+++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb
@@ -3,13 +3,13 @@
 require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb')
 
-describe MigrateUserActivitiesToUsersLastActivityOn, :redis, :truncate do
+describe MigrateUserActivitiesToUsersLastActivityOn, :clean_gitlab_redis_shared_state, :truncate do
   let(:migration) { described_class.new }
   let!(:user_active_1) { create(:user) }
   let!(:user_active_2) { create(:user) }
 
   def record_activity(user, time)
-    Gitlab::Redis.with do |redis|
+    Gitlab::Redis::SharedState.with do |redis|
       redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username)
     end
   end
diff --git a/spec/migrations/move_personal_snippets_files_spec.rb b/spec/migrations/move_personal_snippets_files_spec.rb
new file mode 100644
index 00000000000..8505c7bf3e3
--- /dev/null
+++ b/spec/migrations/move_personal_snippets_files_spec.rb
@@ -0,0 +1,180 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170612071012_move_personal_snippets_files.rb')
+
+describe MovePersonalSnippetsFiles do
+  let(:migration) { described_class.new }
+  let(:test_dir) { File.join(Rails.root, "tmp", "tests", "move_snippet_files_test") }
+  let(:uploads_dir) { File.join(test_dir, 'uploads') }
+  let(:new_uploads_dir) { File.join(uploads_dir, 'system') }
+
+  before do
+    allow(CarrierWave).to receive(:root).and_return(test_dir)
+    allow(migration).to receive(:base_directory).and_return(test_dir)
+    FileUtils.remove_dir(test_dir) if File.directory?(test_dir)
+    allow(migration).to receive(:say)
+  end
+
+  describe "#up" do
+    let(:snippet) do
+      snippet = create(:personal_snippet)
+      create_upload('picture.jpg', snippet)
+      snippet.update(description: markdown_linking_file('picture.jpg', snippet))
+      snippet
+    end
+
+    let(:snippet_with_missing_file) do
+      snippet = create(:snippet)
+      create_upload('picture.jpg', snippet, create_file: false)
+      snippet.update(description: markdown_linking_file('picture.jpg', snippet))
+      snippet
+    end
+
+    it 'moves the files' do
+      source_path = File.join(uploads_dir, model_file_path('picture.jpg', snippet))
+      destination_path = File.join(new_uploads_dir, model_file_path('picture.jpg', snippet))
+
+      migration.up
+
+      expect(File.exist?(source_path)).to be_falsy
+      expect(File.exist?(destination_path)).to be_truthy
+    end
+
+    describe 'updating the markdown' do
+      it 'includes the new path when the file exists' do
+        secret = "secret#{snippet.id}"
+        file_location = "/uploads/system/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
+
+        migration.up
+
+        expect(snippet.reload.description).to include(file_location)
+      end
+
+      it 'does not update the markdown when the file is missing' do
+        secret = "secret#{snippet_with_missing_file.id}"
+        file_location = "/uploads/personal_snippet/#{snippet_with_missing_file.id}/#{secret}/picture.jpg"
+
+        migration.up
+
+        expect(snippet_with_missing_file.reload.description).to include(file_location)
+      end
+
+      it 'updates the note markdown' do
+        secret = "secret#{snippet.id}"
+        file_location = "/uploads/system/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
+        markdown = markdown_linking_file('picture.jpg', snippet)
+        note = create(:note_on_personal_snippet, noteable: snippet, note: "with #{markdown}")
+
+        migration.up
+
+        expect(note.reload.note).to include(file_location)
+      end
+    end
+  end
+
+  describe "#down" do
+    let(:snippet) do
+      snippet = create(:personal_snippet)
+      create_upload('picture.jpg', snippet, in_new_path: true)
+      snippet.update(description: markdown_linking_file('picture.jpg', snippet, in_new_path: true))
+      snippet
+    end
+
+    let(:snippet_with_missing_file) do
+      snippet = create(:personal_snippet)
+      create_upload('picture.jpg', snippet, create_file: false, in_new_path: true)
+      snippet.update(description: markdown_linking_file('picture.jpg', snippet, in_new_path: true))
+      snippet
+    end
+
+    it 'moves the files' do
+      source_path = File.join(new_uploads_dir, model_file_path('picture.jpg', snippet))
+      destination_path = File.join(uploads_dir, model_file_path('picture.jpg', snippet))
+
+      migration.down
+
+      expect(File.exist?(source_path)).to be_falsey
+      expect(File.exist?(destination_path)).to be_truthy
+    end
+
+    describe 'updating the markdown' do
+      it 'includes the new path when the file exists' do
+        secret = "secret#{snippet.id}"
+        file_location = "/uploads/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
+
+        migration.down
+
+        expect(snippet.reload.description).to include(file_location)
+      end
+
+      it 'keeps the markdown as is when the file is missing' do
+        secret = "secret#{snippet_with_missing_file.id}"
+        file_location = "/uploads/system/personal_snippet/#{snippet_with_missing_file.id}/#{secret}/picture.jpg"
+
+        migration.down
+
+        expect(snippet_with_missing_file.reload.description).to include(file_location)
+      end
+
+      it 'updates the note markdown' do
+        markdown = markdown_linking_file('picture.jpg', snippet, in_new_path: true)
+        secret = "secret#{snippet.id}"
+        file_location = "/uploads/personal_snippet/#{snippet.id}/#{secret}/picture.jpg"
+        note = create(:note_on_personal_snippet, noteable: snippet, note: "with #{markdown}")
+
+        migration.down
+
+        expect(note.reload.note).to include(file_location)
+      end
+    end
+  end
+
+  describe '#update_markdown' do
+    it 'escapes sql in the snippet description' do
+      migration.instance_variable_set('@source_relative_location', '/uploads/personal_snippet')
+      migration.instance_variable_set('@destination_relative_location', '/uploads/system/personal_snippet')
+
+      secret = '123456789'
+      filename = 'hello.jpg'
+      snippet = create(:personal_snippet)
+
+      path_before = "/uploads/personal_snippet/#{snippet.id}/#{secret}/#{filename}"
+      path_after = "/uploads/system/personal_snippet/#{snippet.id}/#{secret}/#{filename}"
+      description_before = "Hello world; ![image](#{path_before})'; select * from users;"
+      description_after = "Hello world; ![image](#{path_after})'; select * from users;"
+
+      migration.update_markdown(snippet.id, secret, filename, description_before)
+
+      expect(snippet.reload.description).to eq(description_after)
+    end
+  end
+
+  def create_upload(filename, snippet, create_file: true, in_new_path: false)
+    secret = "secret#{snippet.id}"
+    absolute_path = if in_new_path
+                      File.join(new_uploads_dir, model_file_path(filename, snippet))
+                    else
+                      File.join(uploads_dir, model_file_path(filename, snippet))
+                    end
+
+    if create_file
+      FileUtils.mkdir_p(File.dirname(absolute_path))
+      FileUtils.touch(absolute_path)
+    end
+
+    create(:upload, model: snippet, path: "#{secret}/#{filename}", uploader: PersonalFileUploader)
+  end
+
+  def markdown_linking_file(filename, snippet, in_new_path: false)
+    markdown =  "![#{filename.split('.')[0]}]"
+    markdown += '(/uploads'
+    markdown += '/system' if in_new_path
+    markdown += "/#{model_file_path(filename, snippet)})"
+    markdown
+  end
+
+  def model_file_path(filename, snippet)
+    secret = "secret#{snippet.id}"
+
+    File.join('personal_snippet', snippet.id.to_s, secret, filename)
+  end
+end
diff --git a/spec/migrations/move_system_upload_folder_spec.rb b/spec/migrations/move_system_upload_folder_spec.rb
new file mode 100644
index 00000000000..b622b4e9536
--- /dev/null
+++ b/spec/migrations/move_system_upload_folder_spec.rb
@@ -0,0 +1,62 @@
+require 'spec_helper'
+require Rails.root.join("db", "migrate", "20170717074009_move_system_upload_folder.rb")
+
+describe MoveSystemUploadFolder do
+  let(:migration) { described_class.new }
+  let(:test_base) { File.join(Rails.root, 'tmp', 'tests', 'move-system-upload-folder') }
+
+  before do
+    allow(migration).to receive(:base_directory).and_return(test_base)
+    FileUtils.rm_rf(test_base)
+    FileUtils.mkdir_p(test_base)
+    allow(migration).to receive(:say)
+  end
+
+  describe '#up' do
+    let(:test_folder) { File.join(test_base, 'system') }
+    let(:test_file) { File.join(test_folder, 'file') }
+
+    before do
+      FileUtils.mkdir_p(test_folder)
+      FileUtils.touch(test_file)
+    end
+
+    it 'moves the related folder' do
+      migration.up
+
+      expect(File.exist?(File.join(test_base, '-', 'system', 'file'))).to be_truthy
+    end
+
+    it 'creates a symlink linking making the new folder available on the old path' do
+      migration.up
+
+      expect(File.symlink?(File.join(test_base, 'system'))).to be_truthy
+      expect(File.exist?(File.join(test_base, 'system', 'file'))).to be_truthy
+    end
+  end
+
+  describe '#down' do
+    let(:test_folder) { File.join(test_base, '-', 'system') }
+    let(:test_file) { File.join(test_folder, 'file') }
+
+    before do
+      FileUtils.mkdir_p(test_folder)
+      FileUtils.touch(test_file)
+    end
+
+    it 'moves the system folder back to the old location' do
+      migration.down
+
+      expect(File.exist?(File.join(test_base, 'system', 'file'))).to be_truthy
+    end
+
+    it 'removes the symlink if it existed' do
+      FileUtils.ln_s(test_folder, File.join(test_base, 'system'))
+
+      migration.down
+
+      expect(File.directory?(File.join(test_base, 'system'))).to be_truthy
+      expect(File.symlink?(File.join(test_base, 'system'))).to be_falsey
+    end
+  end
+end