Merge branch 'master' into list-multiple-clusters

* master: (158 commits)
  Improve output for extra queries in specs
  Fixed new group milestone breadcrumb
  Try to find the merge-base against the canonical master
  Add FetchSourceBranch Gitaly call
  Backport QA code that belongs to CE from EE Geo
  Add QUERY_RECORDER_DEBUG environment variable to improve performance debugging
  Add support of Mermaid
  Update VERSION to 10.3.0-pre
  Update CHANGELOG.md for 10.2.0
  default fill color for SVGs
  Fix reply quote keyboard shortcut on MRs
  ignore hashed repos (for now) when using `rake gitlab:cleanup:repos`
  Use Redis cache for branch existence checks
  Update CONTRIBUTING.md: Link definition of done to criteria
  Use `make install` for Gitaly setups in non-test environments
  FileUploader should check for hashed_storage?(:attachments) to use disk_path
  Set the default gitlab-shell timeout to 3 hours
  Update composite pipelines index to include "id"
  Use arrays in Pipeline#latest_builds_with_artifacts
  Fix blank states using old css
  ...

15 files changed, 128 insertions, 46 deletions

diff --git a/app/services/base_count_service.rb b/app/services/base_count_service.rb
new file mode 100644
index 00000000000..99cc9a196e6
--- /dev/null
+++ b/app/services/base_count_service.rb
@@ -0,0 +1,34 @@
+# Base class for services that count a single resource such as the number of
+# issues for a project.
+class BaseCountService
+  def relation_for_count
+    raise(
+      NotImplementedError,
+      '"relation_for_count" must be implemented and return an ActiveRecord::Relation'
+    )
+  end
+
+  def count
+    Rails.cache.fetch(cache_key, raw: raw?) { uncached_count }.to_i
+  end
+
+  def refresh_cache
+    Rails.cache.write(cache_key, uncached_count, raw: raw?)
+  end
+
+  def uncached_count
+    relation_for_count.count
+  end
+
+  def delete_cache
+    Rails.cache.delete(cache_key)
+  end
+
+  def raw?
+    false
+  end
+
+  def cache_key
+    raise NotImplementedError, 'cache_key must be implemented and return a String'
+  end
+end
diff --git a/app/services/ci/fetch_kubernetes_token_service.rb b/app/services/ci/fetch_kubernetes_token_service.rb
index 44da87cb00c..e73c6ad6780 100644
--- a/app/services/ci/fetch_kubernetes_token_service.rb
+++ b/app/services/ci/fetch_kubernetes_token_service.rb
@@ -34,6 +34,7 @@ module Ci
       kubeclient.get_secrets.as_json
     rescue KubeException => err
       raise err unless err.error_code == 404
+
       []
     end
 
diff --git a/app/services/labels/promote_service.rb b/app/services/labels/promote_service.rb
index 43b539ded53..997d247be46 100644
--- a/app/services/labels/promote_service.rb
+++ b/app/services/labels/promote_service.rb
@@ -19,6 +19,7 @@ module Labels
 
         # We skipped validations during creation. Let's run them now, after deleting conflicting labels
         raise ActiveRecord::RecordInvalid.new(new_label) unless new_label.valid?
+
         new_label
       end
     end
diff --git a/app/services/merge_requests/build_service.rb b/app/services/merge_requests/build_service.rb
index bc0e7ad4e39..f3b99e1ec8c 100644
--- a/app/services/merge_requests/build_service.rb
+++ b/app/services/merge_requests/build_service.rb
@@ -28,6 +28,7 @@ module MergeRequests
 
     def find_target_project
       return target_project if target_project.present? && can?(current_user, :read_project, target_project)
+
       project.default_merge_request_target
     end
 
diff --git a/app/services/merge_requests/merge_service.rb b/app/services/merge_requests/merge_service.rb
index 1da4dbd9e96..cedfcb50e09 100644
--- a/app/services/merge_requests/merge_service.rb
+++ b/app/services/merge_requests/merge_service.rb
@@ -10,6 +10,8 @@ module MergeRequests
 
     attr_reader :merge_request, :source
 
+    delegate :merge_jid, :state, to: :@merge_request
+
     def execute(merge_request)
       if project.merge_requests_ff_only_enabled && !self.is_a?(FfMergeService)
         FfMergeService.new(project, current_user, params).execute(merge_request)
@@ -27,6 +29,7 @@ module MergeRequests
           success
         end
       end
+      log_info("Merge process finished on JID #{merge_jid} with state #{state}")
     rescue MergeError => e
       handle_merge_error(log_message: e.message, save_message_on_model: true)
     end
@@ -49,7 +52,9 @@ module MergeRequests
     def commit
       message = params[:commit_message] || merge_request.merge_commit_message
 
+      log_info("Git merge started on JID #{merge_jid}")
       commit_id = repository.merge(current_user, source, merge_request, message)
+      log_info("Git merge finished on JID #{merge_jid} commit #{commit_id}")
 
       raise MergeError, 'Conflicts detected during merge' unless commit_id
 
@@ -63,7 +68,9 @@ module MergeRequests
     end
 
     def after_merge
+      log_info("Post merge started on JID #{merge_jid} with state #{state}")
       MergeRequests::PostMergeService.new(project, current_user).execute(merge_request)
+      log_info("Post merge finished on JID #{merge_jid} with state #{state}")
 
       if delete_source_branch?
         DeleteBranchService.new(@merge_request.source_project, branch_deletion_user)
@@ -92,6 +99,11 @@ module MergeRequests
       @merge_request.update(merge_error: log_message) if save_message_on_model
     end
 
+    def log_info(message)
+      @logger ||= Rails.logger
+      @logger.info("#{merge_request_info} - #{message}")
+    end
+
     def merge_request_info
       merge_request.to_reference(full: true)
     end
diff --git a/app/services/milestones/promote_service.rb b/app/services/milestones/promote_service.rb
index bd9cfd4e0ea..2187f26d1ed 100644
--- a/app/services/milestones/promote_service.rb
+++ b/app/services/milestones/promote_service.rb
@@ -6,14 +6,14 @@ module Milestones
       check_project_milestone!(milestone)
 
       Milestone.transaction do
-        # Destroy all milestones with same title across projects
-        destroy_old_milestones(milestone)
-
         group_milestone = clone_project_milestone(milestone)
 
         move_children_to_group_milestone(group_milestone)
 
-        # Just to be safe
+        # Destroy all milestones with same title across projects
+        destroy_old_milestones(milestone)
+
+        # Rollback if milestone is not valid
         unless group_milestone.valid?
           raise_error(group_milestone.errors.full_messages.to_sentence)
         end
@@ -35,7 +35,7 @@ module Milestones
     end
 
     def move_children_to_group_milestone(group_milestone)
-      milestone_ids_for_merge(group_milestone).in_groups_of(100) do |milestone_ids|
+      milestone_ids_for_merge(group_milestone).in_groups_of(100, false) do |milestone_ids|
         update_children(group_milestone, milestone_ids)
       end
     end
@@ -49,7 +49,12 @@ module Milestones
 
       create_service = CreateService.new(group, current_user, params)
 
-      create_service.execute
+      milestone = create_service.execute
+
+      # milestone won't be valid here because of duplicated title
+      milestone.save(validate: false)
+
+      milestone
     end
 
     def update_children(group_milestone, milestone_ids)
@@ -65,12 +70,12 @@ module Milestones
       @group ||= parent.group || raise_error('Project does not belong to a group.')
     end
 
-    def destroy_old_milestones(group_milestone)
-      Milestone.where(id: milestone_ids_for_merge(group_milestone)).destroy_all
+    def destroy_old_milestones(milestone)
+      Milestone.where(id: milestone_ids_for_merge(milestone)).destroy_all
     end
 
     def group_project_ids
-      @group_project_ids ||= group.projects.map(&:id)
+      @group_project_ids ||= group.projects.pluck(:id)
     end
 
     def raise_error(message)
diff --git a/app/services/projects/count_service.rb b/app/services/projects/count_service.rb
index aa034315280..7e575b2d6f3 100644
--- a/app/services/projects/count_service.rb
+++ b/app/services/projects/count_service.rb
@@ -1,7 +1,7 @@
 module Projects
   # Base class for the various service classes that count project data (e.g.
   # issues or forks).
-  class CountService
+  class CountService < BaseCountService
     # The version of the cache format. This should be bumped whenever the
     # underlying logic changes. This removes the need for explicitly flushing
     # all caches.
@@ -11,29 +11,6 @@ module Projects
       @project = project
     end
 
-    def relation_for_count
-      raise(
-        NotImplementedError,
-        '"relation_for_count" must be implemented and return an ActiveRecord::Relation'
-      )
-    end
-
-    def count
-      Rails.cache.fetch(cache_key) { uncached_count }
-    end
-
-    def refresh_cache
-      Rails.cache.write(cache_key, uncached_count)
-    end
-
-    def uncached_count
-      relation_for_count.count
-    end
-
-    def delete_cache
-      Rails.cache.delete(cache_key)
-    end
-
     def cache_key_name
       raise(
         NotImplementedError,
diff --git a/app/services/projects/forks_count_service.rb b/app/services/projects/forks_count_service.rb
index 3a0fa84b868..d9bdf3a8ad7 100644
--- a/app/services/projects/forks_count_service.rb
+++ b/app/services/projects/forks_count_service.rb
@@ -1,6 +1,6 @@
 module Projects
   # Service class for getting and caching the number of forks of a project.
-  class ForksCountService < CountService
+  class ForksCountService < Projects::CountService
     def relation_for_count
       @project.forks
     end
diff --git a/app/services/projects/group_links/destroy_service.rb b/app/services/projects/group_links/destroy_service.rb
index fbf31214c28..e3a20b4c1e4 100644
--- a/app/services/projects/group_links/destroy_service.rb
+++ b/app/services/projects/group_links/destroy_service.rb
@@ -3,6 +3,7 @@ module Projects
     class DestroyService < BaseService
       def execute(group_link)
         return false unless group_link
+
         group_link.destroy
       end
     end
diff --git a/app/services/projects/open_issues_count_service.rb b/app/services/projects/open_issues_count_service.rb
index 3c0d186a73c..25de97325e2 100644
--- a/app/services/projects/open_issues_count_service.rb
+++ b/app/services/projects/open_issues_count_service.rb
@@ -1,7 +1,7 @@
 module Projects
   # Service class for counting and caching the number of open issues of a
   # project.
-  class OpenIssuesCountService < CountService
+  class OpenIssuesCountService < Projects::CountService
     def relation_for_count
       # We don't include confidential issues in this number since this would
       # expose the number of confidential issues to non project members.
diff --git a/app/services/projects/open_merge_requests_count_service.rb b/app/services/projects/open_merge_requests_count_service.rb
index 2a90f78b90d..77e6448fd5e 100644
--- a/app/services/projects/open_merge_requests_count_service.rb
+++ b/app/services/projects/open_merge_requests_count_service.rb
@@ -1,7 +1,7 @@
 module Projects
   # Service class for counting and caching the number of open merge requests of
   # a project.
-  class OpenMergeRequestsCountService < CountService
+  class OpenMergeRequestsCountService < Projects::CountService
     def relation_for_count
       @project.merge_requests.opened
     end
diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb
index 5957f612e84..e5cd6fcdfe3 100644
--- a/app/services/projects/transfer_service.rb
+++ b/app/services/projects/transfer_service.rb
@@ -60,21 +60,14 @@ module Projects
         # Notifications
         project.send_move_instructions(@old_path)
 
-        # Move main repository
-        # TODO: check storage type and NOOP when not using Legacy
-        unless move_repo_folder(@old_path, @new_path)
-          raise TransferError.new('Cannot move project')
-        end
-
-        # Move wiki repo also if present
-        # TODO: check storage type and NOOP when not using Legacy
-        move_repo_folder("#{@old_path}.wiki", "#{@new_path}.wiki")
+        # Directories on disk
+        move_project_folders(project)
 
         # Move missing group labels to project
         Labels::TransferService.new(current_user, @old_group, project).execute
 
         # Move uploads
-        Gitlab::UploadsTransfer.new.move_project(project.path, @old_namespace.full_path, @new_namespace.full_path)
+        move_project_uploads(project)
 
         # Move pages
         Gitlab::PagesTransfer.new.move_project(project.path, @old_namespace.full_path, @new_namespace.full_path)
@@ -131,5 +124,30 @@ module Projects
     def execute_system_hooks
       SystemHooksService.new.execute_hooks_for(project, :transfer)
     end
+
+    def move_project_folders(project)
+      return if project.hashed_storage?(:repository)
+
+      # Move main repository
+      unless move_repo_folder(@old_path, @new_path)
+        raise TransferError.new("Cannot move project")
+      end
+
+      # Disk path is changed; we need to ensure we reload it
+      project.reload_repository!
+
+      # Move wiki repo also if present
+      move_repo_folder("#{@old_path}.wiki", "#{@new_path}.wiki")
+    end
+
+    def move_project_uploads(project)
+      return if project.hashed_storage?(:attachments)
+
+      Gitlab::UploadsTransfer.new.move_project(
+        project.path,
+        @old_namespace.full_path,
+        @new_namespace.full_path
+      )
+    end
   end
 end
diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb
index e946218824c..fe71a405565 100644
--- a/app/services/system_note_service.rb
+++ b/app/services/system_note_service.rb
@@ -583,6 +583,10 @@ module SystemNoteService
     create_note(NoteSummary.new(issuable, issuable.project, author, body, action: action))
   end
 
+  def cross_reference?(note_text)
+    note_text =~ /\A#{cross_reference_note_prefix}/i
+  end
+
   private
 
   def notes_for_mentioner(mentioner, noteable, notes)
diff --git a/app/services/todo_service.rb b/app/services/todo_service.rb
index e694c5761da..575853fd66b 100644
--- a/app/services/todo_service.rb
+++ b/app/services/todo_service.rb
@@ -208,6 +208,7 @@ class TodoService
   def create_todos(users, attributes)
     Array(users).map do |user|
       next if pending_todos(user, attributes).exists?
+
       todo = Todo.create(attributes.merge(user_id: user.id))
       user.update_todos_count_cache
       todo
diff --git a/app/services/users/keys_count_service.rb b/app/services/users/keys_count_service.rb
new file mode 100644
index 00000000000..f82d27eded9
--- /dev/null
+++ b/app/services/users/keys_count_service.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Users
+  # Service class for getting the number of SSH keys that belong to a user.
+  class KeysCountService < BaseCountService
+    attr_reader :user
+
+    # user - The User for which to get the number of SSH keys.
+    def initialize(user)
+      @user = user
+    end
+
+    def relation_for_count
+      user.keys
+    end
+
+    def raw?
+      # Since we're storing simple integers we don't need all of the additional
+      # Marshal data Rails includes by default.
+      true
+    end
+
+    def cache_key
+      "users/key-count-service/#{user.id}"
+    end
+  end
+end