Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (507 commits)
  Add dropdowns documentation
  Convert migration to populate latest merge request ID into a background migration
  Set 0.69.0 instead of latest for codeclimate image
  De-duplicate background migration matchers defined in spec/support/migrations_helpers.rb
  Update database_debugging.md
  Update database_debugging.md
  Move installation of apps higher
  Change to Google Kubernetes Cluster and add internal links
  Add Ingress description from official docs
  Add info on creating your own k8s cluster from the cluster page
  Add info about the installed apps in the Cluster docs
  Resolve "lock/confidential issuable sidebar custom svg icons iteration"
  Update HA README.md to clarify GitLab support does not troubleshoot DRBD.
  Update license_finder to 3.1.1
  Make sure NotesActions#noteable returns a Noteable in the update action
  Cache the number of user SSH keys
  Adjust openid_connect_spec to use `raise_error`
  Resolve "Clicking on GPG verification badge jumps to top of the page"
  Add changelog for container repository path update
  Update container repository path reference
  ...

6 files changed, 180 insertions, 0 deletions

diff --git a/app/workers/concerns/cluster_applications.rb b/app/workers/concerns/cluster_applications.rb
new file mode 100644
index 00000000000..24ecaa0b52f
--- /dev/null
+++ b/app/workers/concerns/cluster_applications.rb
@@ -0,0 +1,9 @@
+module ClusterApplications
+  extend ActiveSupport::Concern
+
+  included do
+    def find_application(app_name, id, &blk)
+      Clusters::Cluster::APPLICATIONS[app_name].find(id).try(&blk)
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/github_import/notify_upon_death.rb b/app/workers/concerns/gitlab/github_import/notify_upon_death.rb
new file mode 100644
index 00000000000..3d7120665b6
--- /dev/null
+++ b/app/workers/concerns/gitlab/github_import/notify_upon_death.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    # NotifyUponDeath can be included into a GitHub worker class if it should
+    # notify any JobWaiter instances upon being moved to the Sidekiq dead queue.
+    #
+    # Note that this will only notify the waiter upon graceful termination, a
+    # SIGKILL will still result in the waiter _not_ being notified.
+    #
+    # Workers including this module must have jobs passed where the last
+    # argument is the key to notify, as a String.
+    module NotifyUponDeath
+      extend ActiveSupport::Concern
+
+      included do
+        # If a job is being exhausted we still want to notify the
+        # AdvanceStageWorker. This prevents the entire import from getting stuck
+        # just because 1 job threw too many errors.
+        sidekiq_retries_exhausted do |job|
+          args = job['args']
+          jid = job['jid']
+
+          if args.length == 3 && (key = args.last) && key.is_a?(String)
+            JobWaiter.notify(key, jid)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb
new file mode 100644
index 00000000000..67e36c811de
--- /dev/null
+++ b/app/workers/concerns/gitlab/github_import/object_importer.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    # ObjectImporter defines the base behaviour for every Sidekiq worker that
+    # imports a single resource such as a note or pull request.
+    module ObjectImporter
+      extend ActiveSupport::Concern
+
+      included do
+        include Sidekiq::Worker
+        include GithubImport::Queue
+        include ReschedulingMethods
+        include NotifyUponDeath
+      end
+
+      # project - An instance of `Project` to import the data into.
+      # client - An instance of `Gitlab::GithubImport::Client`
+      # hash - A Hash containing the details of the object to import.
+      def import(project, client, hash)
+        object = representation_class.from_json_hash(hash)
+
+        importer_class.new(object, project, client).execute
+
+        counter.increment(project: project.path_with_namespace)
+      end
+
+      def counter
+        @counter ||= Gitlab::Metrics.counter(counter_name, counter_description)
+      end
+
+      # Returns the representation class to use for the object. This class must
+      # define the class method `from_json_hash`.
+      def representation_class
+        raise NotImplementedError
+      end
+
+      # Returns the class to use for importing the object.
+      def importer_class
+        raise NotImplementedError
+      end
+
+      # Returns the name (as a Symbol) of the Prometheus counter.
+      def counter_name
+        raise NotImplementedError
+      end
+
+      # Returns the description (as a String) of the Prometheus counter.
+      def counter_description
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb
new file mode 100644
index 00000000000..a2bee361b86
--- /dev/null
+++ b/app/workers/concerns/gitlab/github_import/queue.rb
@@ -0,0 +1,16 @@
+module Gitlab
+  module GithubImport
+    module Queue
+      extend ActiveSupport::Concern
+
+      included do
+        # If a job produces an error it may block a stage from advancing
+        # forever. To prevent this from happening we prevent jobs from going to
+        # the dead queue. This does mean some resources may not be imported, but
+        # this is better than a project being stuck in the "import" state
+        # forever.
+        sidekiq_options queue: 'github_importer', dead: false, retry: 5
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
new file mode 100644
index 00000000000..692ca6b7f42
--- /dev/null
+++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    # Module that provides methods shared by the various workers used for
+    # importing GitHub projects.
+    module ReschedulingMethods
+      # project_id - The ID of the GitLab project to import the note into.
+      # hash - A Hash containing the details of the GitHub object to imoprt.
+      # notify_key - The Redis key to notify upon completion, if any.
+      def perform(project_id, hash, notify_key = nil)
+        project = Project.find_by(id: project_id)
+
+        return notify_waiter(notify_key) unless project
+
+        client = GithubImport.new_client_for(project, parallel: true)
+
+        if try_import(project, client, hash)
+          notify_waiter(notify_key)
+        else
+          # In the event of hitting the rate limit we want to reschedule the job
+          # so its retried after our rate limit has been reset.
+          self.class
+            .perform_in(client.rate_limit_resets_in, project.id, hash, notify_key)
+        end
+      end
+
+      def try_import(*args)
+        import(*args)
+        true
+      rescue RateLimitError
+        false
+      end
+
+      def notify_waiter(key = nil)
+        JobWaiter.notify(key, jid) if key
+      end
+    end
+  end
+end
diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb
new file mode 100644
index 00000000000..147c8c8d683
--- /dev/null
+++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    module StageMethods
+      # project_id - The ID of the GitLab project to import the data into.
+      def perform(project_id)
+        return unless (project = find_project(project_id))
+
+        client = GithubImport.new_client_for(project)
+
+        try_import(client, project)
+      end
+
+      # client - An instance of Gitlab::GithubImport::Client.
+      # project - An instance of Project.
+      def try_import(client, project)
+        import(client, project)
+      rescue RateLimitError
+        self.class.perform_in(client.rate_limit_resets_in, project.id)
+      end
+
+      def find_project(id)
+        # If the project has been marked as failed we want to bail out
+        # automatically.
+        Project.import_started.find_by(id: id)
+      end
+    end
+  end
+end