Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (507 commits)
  Add dropdowns documentation
  Convert migration to populate latest merge request ID into a background migration
  Set 0.69.0 instead of latest for codeclimate image
  De-duplicate background migration matchers defined in spec/support/migrations_helpers.rb
  Update database_debugging.md
  Update database_debugging.md
  Move installation of apps higher
  Change to Google Kubernetes Cluster and add internal links
  Add Ingress description from official docs
  Add info on creating your own k8s cluster from the cluster page
  Add info about the installed apps in the Cluster docs
  Resolve "lock/confidential issuable sidebar custom svg icons iteration"
  Update HA README.md to clarify GitLab support does not troubleshoot DRBD.
  Update license_finder to 3.1.1
  Make sure NotesActions#noteable returns a Noteable in the update action
  Cache the number of user SSH keys
  Adjust openid_connect_spec to use `raise_error`
  Resolve "Clicking on GPG verification badge jumps to top of the page"
  Add changelog for container repository path update
  Update container repository path reference
  ...

4 files changed, 79 insertions, 43 deletions

diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index 87ca39b079b..c2d3a6b6950 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -1,5 +1,28 @@
 namespace :gemojione do
   desc 'Generates Emoji SHA256 digests'
+
+  task aliases: ['yarn:check', 'environment'] do
+    require 'json'
+
+    aliases = {}
+
+    index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json')
+    index = JSON.parse(File.read(index_file))
+
+    index.each_pair do |key, data|
+      data['aliases'].each do |a|
+        a.tr!(':', '')
+
+        aliases[a] = key
+      end
+    end
+
+    out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
+    File.open(out, 'w') do |handle|
+      handle.write(JSON.pretty_generate(aliases, indent: '   ', space: '', space_before: ''))
+    end
+  end
+
   task digests: ['yarn:check', 'environment'] do
     require 'digest/sha2'
     require 'json'
@@ -16,8 +39,13 @@ namespace :gemojione do
         fpath = File.join(dir, "#{emoji_hash['unicode']}.png")
         hash_digest = Digest::SHA256.file(fpath).hexdigest
 
+        category = emoji_hash['category']
+        if name == 'gay_pride_flag'
+          category = 'flags'
+        end
+
         entry = {
-          category: emoji_hash['category'],
+          category: category,
           moji: emoji_hash['moji'],
           description: emoji_hash['description'],
           unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name),
@@ -29,7 +57,6 @@ namespace :gemojione do
     end
 
     out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
-
     File.open(out, 'w') do |handle|
       handle.write(JSON.pretty_generate(resultant_emoji_map))
     end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 1650263b98d..9dcf44fdc3e 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -33,24 +33,29 @@ namespace :gitlab do
       backup.unpack
 
       unless backup.skipped?('db')
-        unless ENV['force'] == 'yes'
-          warning = <<-MSG.strip_heredoc
-            Before restoring the database we recommend removing all existing
-            tables to avoid future upgrade problems. Be aware that if you have
-            custom tables in the GitLab database these tables and all data will be
-            removed.
-          MSG
-          puts warning.color(:red)
-          ask_to_continue
-          puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
-          sleep(5)
+        begin
+          unless ENV['force'] == 'yes'
+            warning = <<-MSG.strip_heredoc
+              Before restoring the database, we will remove all existing
+              tables to avoid future upgrade problems. Be aware that if you have
+              custom tables in the GitLab database these tables and all data will be
+              removed.
+            MSG
+            puts warning.color(:red)
+            ask_to_continue
+            puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
+            sleep(5)
+          end
+          # Drop all tables Load the schema to ensure we don't have any newer tables
+          # hanging out from a failed upgrade
+          $progress.puts 'Cleaning the database ... '.color(:blue)
+          Rake::Task['gitlab:db:drop_tables'].invoke
+          $progress.puts 'done'.color(:green)
+          Rake::Task['gitlab:backup:db:restore'].invoke
+        rescue Gitlab::TaskAbortedByUserError
+          puts "Quitting...".color(:red)
+          exit 1
         end
-        # Drop all tables Load the schema to ensure we don't have any newer tables
-        # hanging out from a failed upgrade
-        $progress.puts 'Cleaning the database ... '.color(:blue)
-        Rake::Task['gitlab:db:drop_tables'].invoke
-        $progress.puts 'done'.color(:green)
-        Rake::Task['gitlab:backup:db:restore'].invoke
       end
 
       Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories')
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index d227a0c8bdb..adfcc3cda22 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -2,23 +2,21 @@ namespace :gitlab do
   namespace :import do
     # How to use:
     #
-    #  1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories)
-    #  2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production
+    #  1. copy the bare repos to a specific path that contain the group or subgroups structure as folders
+    #  2. run: bundle exec rake gitlab:import:repos[/path/to/repos] RAILS_ENV=production
     #
     # Notes:
     #  * The project owner will set to the first administator of the system
     #  * Existing projects will be skipped
-    #
-    #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
-    task repos: :environment do
-      if Project.current_application_settings.hashed_storage_enabled
-        puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red)
+    task :repos, [:import_path] => :environment do |_t, args|
+      unless args.import_path
+        puts 'Please specify an import path that contains the repositories'.color(:red)
 
         exit 1
       end
 
-      Gitlab::BareRepositoryImporter.execute
+      Gitlab::BareRepositoryImport::Importer.execute(args.import_path)
     end
   end
 end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 7f86fd7b45e..aafbe52e5f8 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -7,14 +7,16 @@ class GithubImport
   end
 
   def initialize(token, gitlab_username, project_path, extras)
-    @options = { token: token, verbose: true }
+    @options = { token: token }
     @project_path = project_path
     @current_user = User.find_by_username(gitlab_username)
     @github_repo = extras.empty? ? nil : extras.first
   end
 
   def run!
-    @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one!
+    @repo = GithubRepos
+      .new(@options[:token], @current_user, @github_repo)
+      .choose_one!
 
     raise 'No repo found!' unless @repo
 
@@ -28,7 +30,7 @@ class GithubImport
   private
 
   def show_warning!
-    puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}"
+    puts "This will import GitHub #{@repo.full_name.bright} into GitLab #{@project_path.bright} as #{@current_user.name}"
     puts "Permission checks are ignored. Press any key to continue.".color(:red)
 
     STDIN.getch
@@ -42,7 +44,9 @@ class GithubImport
     import_success = false
 
     timings = Benchmark.measure do
-      import_success = Github::Import.new(@project, @options).execute
+      import_success = Gitlab::GithubImport::SequentialImporter
+        .new(@project, token: @options[:token])
+        .execute
     end
 
     if import_success
@@ -63,16 +67,16 @@ class GithubImport
         @current_user,
         name: name,
         path: name,
-        description: @repo['description'],
+        description: @repo.description,
         namespace_id: namespace.id,
         visibility_level: visibility_level,
-        skip_wiki: @repo['has_wiki']
+        skip_wiki: @repo.has_wiki
       ).execute
 
       project.update!(
         import_type: 'github',
-        import_source: @repo['full_name'],
-        import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@")
+        import_source: @repo.full_name,
+        import_url: @repo.clone_url.sub('://', "://#{@options[:token]}@")
       )
 
       project
@@ -91,13 +95,15 @@ class GithubImport
   end
 
   def visibility_level
-    @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility
+    @repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility
   end
 end
 
 class GithubRepos
-  def initialize(options, current_user, github_repo)
-    @options = options
+  def initialize(token, current_user, github_repo)
+    @client = Gitlab::GithubImport::Client.new(token)
+    @client.octokit.auto_paginate = true
+
     @current_user = current_user
     @github_repo = github_repo
   end
@@ -106,17 +112,17 @@ class GithubRepos
     return found_github_repo if @github_repo
 
     repos.each do |repo|
-      print "ID: #{repo['id'].to_s.bright}".color(:green)
-      print "\tName: #{repo['full_name']}\n".color(:green)
+      print "ID: #{repo.id.to_s.bright}".color(:green)
+      print "\tName: #{repo.full_name}\n".color(:green)
     end
 
     print 'ID? '.bright
 
-    repos.find { |repo| repo['id'] == repo_id }
+    repos.find { |repo| repo.id == repo_id }
   end
 
   def found_github_repo
-    repos.find { |repo| repo['full_name'] == @github_repo }
+    repos.find { |repo| repo.full_name == @github_repo }
   end
 
   def repo_id
@@ -124,7 +130,7 @@ class GithubRepos
   end
 
   def repos
-    Github::Repositories.new(@options).fetch
+    @client.octokit.list_repositories
   end
 end