Merge branch 'master' into backstage/gb/migrate-stages-statuses

* master: (1000 commits)
  Fix username autocomplete group name with no avatar alignment
  Fix 'Projected tags' typo in protected_tags_spec.rb
  Many Repo Fixes
  Repo Editor Fixes
  Docs: New index for permissions
  link article from CI index
  link tech articles from the landing page
  new articles come first
  fix relative link
  fix date format
  Fixed changed files dropdown not being shown
  Update publication date
  Remove deprecated field from workhorse API responses
  Fix API responses when dealing with txt files
  Make sure MySQL would not use CURRENT_TIMESTAMP
  Add two more project templates
  Allow usage of any_projects? with an Array
  Copyedit Artifactory and GitLab article
  Rename Artifactory and GitLab article file
  Display GPG status loading spinner only when Ajax request is made
  ...

10 files changed, 137 insertions, 8 deletions

diff --git a/app/workers/concerns/new_issuable.rb b/app/workers/concerns/new_issuable.rb
new file mode 100644
index 00000000000..eb0d6c9c36c
--- /dev/null
+++ b/app/workers/concerns/new_issuable.rb
@@ -0,0 +1,26 @@
+module NewIssuable
+  attr_reader :issuable, :user
+
+  def objects_found?(issuable_id, user_id)
+    set_user(user_id)
+    set_issuable(issuable_id)
+
+    user && issuable
+  end
+
+  def set_user(user_id)
+    @user = User.find_by(id: user_id)
+
+    log_error(User, user_id) unless @user
+  end
+
+  def set_issuable(issuable_id)
+    @issuable = issuable_class.find_by(id: issuable_id)
+
+    log_error(issuable_class, issuable_id) unless @issuable
+  end
+
+  def log_error(record_class, record_id)
+    Rails.logger.error("#{self.class}: couldn't find #{record_class} with ID=#{record_id}, skipping job")
+  end
+end
diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb
index d3f7e479a8d..1afa24c8e2a 100644
--- a/app/workers/email_receiver_worker.rb
+++ b/app/workers/email_receiver_worker.rb
@@ -31,8 +31,6 @@ class EmailReceiverWorker
       when Gitlab::Email::EmptyEmailError
         can_retry = true
         "It appears that the email is blank. Make sure your reply is at the top of the email, we can't process inline replies."
-      when Gitlab::Email::AutoGeneratedEmailError
-        "The email was marked as 'auto generated', which we can't accept. Please create your comment through the web interface."
       when Gitlab::Email::UserNotFoundError
         "We couldn't figure out what user corresponds to the email. Please create your comment through the web interface."
       when Gitlab::Email::UserBlockedError
diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index d369b639ae9..c95497dfaba 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -5,6 +5,12 @@ class GitGarbageCollectWorker
 
   sidekiq_options retry: false
 
+  GITALY_MIGRATED_TASKS = {
+    gc: :garbage_collect,
+    full_repack: :repack_full,
+    incremental_repack: :repack_incremental
+  }.freeze
+
   def perform(project_id, task = :gc, lease_key = nil, lease_uuid = nil)
     project = Project.find(project_id)
     task = task.to_sym
@@ -15,8 +21,14 @@ class GitGarbageCollectWorker
 
     Gitlab::GitLogger.info(description)
 
-    output, status = Gitlab::Popen.popen(cmd, repo_path)
-    Gitlab::GitLogger.error("#{description} failed:\n#{output}") unless status.zero?
+    gitaly_migrate(GITALY_MIGRATED_TASKS[task]) do |is_enabled|
+      if is_enabled
+        gitaly_call(task, project.repository.raw_repository)
+      else
+        output, status = Gitlab::Popen.popen(cmd, repo_path)
+        Gitlab::GitLogger.error("#{description} failed:\n#{output}") unless status.zero?
+      end
+    end
 
     # Refresh the branch cache in case garbage collection caused a ref lookup to fail
     flush_ref_caches(project) if task == :gc
@@ -26,6 +38,19 @@ class GitGarbageCollectWorker
 
   private
 
+  ## `repository` has to be a Gitlab::Git::Repository
+  def gitaly_call(task, repository)
+    client = Gitlab::GitalyClient::RepositoryService.new(repository)
+    case task
+    when :gc
+      client.garbage_collect(bitmaps_enabled?)
+    when :full_repack
+      client.repack_full(bitmaps_enabled?)
+    when :incremental_repack
+      client.repack_incremental
+    end
+  end
+
   def command(task)
     case task
     when :gc
@@ -55,4 +80,14 @@ class GitGarbageCollectWorker
     config_value = write_bitmaps ? 'true' : 'false'
     %W[git -c repack.writeBitmaps=#{config_value}]
   end
+
+  def gitaly_migrate(method, &block)
+    Gitlab::GitalyClient.migrate(method, &block)
+  rescue GRPC::NotFound => e
+    Gitlab::GitLogger.error("#{method} failed:\nRepository not found")
+    raise Gitlab::Git::Repository::NoRepository.new(e)
+  rescue GRPC::BadStatus => e
+    Gitlab::GitLogger.error("#{method} failed:\n#{e}")
+    raise Gitlab::Git::CommandError.new(e)
+  end
 end
diff --git a/app/workers/irker_worker.rb b/app/workers/irker_worker.rb
index 22f67fa9e9f..3dd14466994 100644
--- a/app/workers/irker_worker.rb
+++ b/app/workers/irker_worker.rb
@@ -66,7 +66,7 @@ class IrkerWorker
   end
 
   def send_new_branch(project, repo_name, committer, branch)
-    repo_path = project.path_with_namespace
+    repo_path = project.full_path
     newbranch = "#{Gitlab.config.gitlab.url}/#{repo_path}/branches"
     newbranch = "\x0302\x1f#{newbranch}\x0f" if @colors
 
@@ -109,7 +109,7 @@ class IrkerWorker
   end
 
   def send_commits_count(data, project, repo, committer, branch)
-    url = compare_url data, project.path_with_namespace
+    url = compare_url data, project.full_path
     commits = colorize_commits data['total_commits_count']
 
     new_commits = 'new commit'
diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb
index 48e2da338f6..c3b58df92c1 100644
--- a/app/workers/merge_worker.rb
+++ b/app/workers/merge_worker.rb
@@ -7,6 +7,8 @@ class MergeWorker
     current_user = User.find(current_user_id)
     merge_request = MergeRequest.find(merge_request_id)
 
+    merge_request.update_column(:merge_jid, jid)
+
     MergeRequests::MergeService.new(merge_request.target_project, current_user, params)
       .execute(merge_request)
   end
diff --git a/app/workers/new_issue_worker.rb b/app/workers/new_issue_worker.rb
new file mode 100644
index 00000000000..d9a8e892e90
--- /dev/null
+++ b/app/workers/new_issue_worker.rb
@@ -0,0 +1,17 @@
+class NewIssueWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+  include NewIssuable
+
+  def perform(issue_id, user_id)
+    return unless objects_found?(issue_id, user_id)
+
+    EventCreateService.new.open_issue(issuable, user)
+    NotificationService.new.new_issue(issuable, user)
+    issuable.create_cross_references!(user)
+  end
+
+  def issuable_class
+    Issue
+  end
+end
diff --git a/app/workers/new_merge_request_worker.rb b/app/workers/new_merge_request_worker.rb
new file mode 100644
index 00000000000..1910c490159
--- /dev/null
+++ b/app/workers/new_merge_request_worker.rb
@@ -0,0 +1,17 @@
+class NewMergeRequestWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+  include NewIssuable
+
+  def perform(merge_request_id, user_id)
+    return unless objects_found?(merge_request_id, user_id)
+
+    EventCreateService.new.open_mr(issuable, user)
+    NotificationService.new.new_merge_request(issuable, user)
+    issuable.create_cross_references!(user)
+  end
+
+  def issuable_class
+    MergeRequest
+  end
+end
diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb
index 4eeb9666bb0..64788da7299 100644
--- a/app/workers/pages_worker.rb
+++ b/app/workers/pages_worker.rb
@@ -4,7 +4,7 @@ class PagesWorker
   sidekiq_options queue: :pages, retry: false
 
   def perform(action, *arg)
-    send(action, *arg)
+    send(action, *arg) # rubocop:disable GitlabSecurity/PublicSend
   end
 
   def deploy(build_id)
diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb
index 625476b7e01..6be541abd3e 100644
--- a/app/workers/repository_import_worker.rb
+++ b/app/workers/repository_import_worker.rb
@@ -16,7 +16,7 @@ class RepositoryImportWorker
 
     Gitlab::Metrics.add_event(:import_repository,
                               import_url: @project.import_url,
-                              path: @project.path_with_namespace)
+                              path: @project.full_path)
 
     project.update_columns(import_jid: self.jid, import_error: nil)
 
diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb
new file mode 100644
index 00000000000..7843179d77c
--- /dev/null
+++ b/app/workers/stuck_merge_jobs_worker.rb
@@ -0,0 +1,34 @@
+class StuckMergeJobsWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  def perform
+    stuck_merge_requests.find_in_batches(batch_size: 100) do |group|
+      jids = group.map(&:merge_jid)
+
+      # Find the jobs that aren't currently running or that exceeded the threshold.
+      completed_jids = Gitlab::SidekiqStatus.completed_jids(jids)
+
+      if completed_jids.any?
+        completed_ids = group.select { |merge_request| completed_jids.include?(merge_request.merge_jid) }.map(&:id)
+
+        apply_current_state!(completed_jids, completed_ids)
+      end
+    end
+  end
+
+  private
+
+  def apply_current_state!(completed_jids, completed_ids)
+    merge_requests = MergeRequest.where(id: completed_ids)
+
+    merge_requests.where.not(merge_commit_sha: nil).update_all(state: :merged)
+    merge_requests.where(merge_commit_sha: nil).update_all(state: :opened)
+
+    Rails.logger.info("Updated state of locked merge jobs. JIDs: #{completed_jids.join(', ')}")
+  end
+
+  def stuck_merge_requests
+    MergeRequest.select('id, merge_jid').with_state(:locked).where.not(merge_jid: nil).reorder(nil)
+  end
+end