Merge branch 'master' into '46448-add-timestamps-for-each-stage-of-gitlab-rake-gitlab-backup-restore'

# Conflicts:
#   spec/tasks/gitlab/backup_rake_spec.rb

25 files changed, 417 insertions, 645 deletions

diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
deleted file mode 100644
index 4b4881cecb8..00000000000
--- a/lib/tasks/flay.rake
+++ /dev/null
@@ -1,9 +0,0 @@
-desc 'Code duplication analyze via flay'
-task :flay do
-  output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}`
-
-  if output.include?("Similar code found") || output.include?("IDENTICAL code found")
-    puts output
-    exit 1
-  end
-end
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index c6942d22926..560a52053d8 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -86,7 +86,7 @@ namespace :gemojione do
     SPRITESHEET_WIDTH = 860
     SPRITESHEET_HEIGHT = 840
 
-    # Setup a map to rename image files
+    # Set up a map to rename image files
     emoji_unicode_string_to_name_map = {}
     Gitlab::Emoji.emojis.each do |name, emoji_hash|
       # Ignore aliases
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index 247d7be7d78..2235a6ba194 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -4,7 +4,7 @@ namespace :gettext do
   # Customize list of translatable files
   # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files
   def files_to_translate
-    folders = %W(app lib config #{locale_path}).join(',')
+    folders = %W(ee app lib config #{locale_path}).join(',')
     exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',')
 
     Dir.glob(
@@ -16,10 +16,32 @@ namespace :gettext do
     # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998
     FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot'))
 
-    Rake::Task['gettext:pack'].invoke
     Rake::Task['gettext:po_to_json'].invoke
   end
 
+  task :regenerate do
+    pot_file = 'locale/gitlab.pot'
+    # Remove all translated files, this speeds up finding
+    FileUtils.rm Dir['locale/**/gitlab.*']
+    # remove the `pot` file to ensure it's completely regenerated
+    FileUtils.rm_f pot_file
+
+    Rake::Task['gettext:find'].invoke
+
+    # leave only the required changes.
+    `git checkout -- locale/*/gitlab.po`
+
+    # Remove timestamps from the pot file
+    pot_content = File.read pot_file
+    pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '')
+    File.write pot_file, pot_content
+
+    puts <<~MSG
+    All done. Please commit the changes to `locale/gitlab.pot`.
+
+    MSG
+  end
+
   desc 'Lint all po files in `locale/'
   task lint: :environment do
     require 'simple_po_parser'
@@ -50,6 +72,40 @@ namespace :gettext do
     end
   end
 
+  task :updated_check do
+    pot_file = 'locale/gitlab.pot'
+    # Removing all pre-translated files speeds up `gettext:find` as the
+    # files don't need to be merged.
+    # Having `LC_MESSAGES/gitlab.mo files present also confuses the output.
+    FileUtils.rm Dir['locale/**/gitlab.*']
+    FileUtils.rm_f pot_file
+
+    # `gettext:find` writes touches to temp files to `stderr` which would cause
+    # `static-analysis` to report failures. We can ignore these.
+    silence_stderr do
+      Rake::Task['gettext:find'].invoke
+    end
+
+    pot_diff = `git diff -- #{pot_file} | grep -E '^(\\+|-)msgid'`.strip
+
+    # reset the locale folder for potential next tasks
+    `git checkout -- locale`
+
+    if pot_diff.present?
+      raise <<~MSG
+        Newly translated strings found, please add them to `#{pot_file}` by running:
+
+          bin/rake gettext:regenerate
+
+        Then commit and push the resulting changes to `#{pot_file}`.
+
+        The diff was:
+
+        #{pot_diff}
+      MSG
+    end
+  end
+
   def report_errors_for_file(file, errors_for_file)
     puts "Errors in `#{file}`:"
 
@@ -62,4 +118,15 @@ namespace :gettext do
       end
     end
   end
+
+  def silence_stderr(&block)
+    old_stderr = $stderr.dup
+    $stderr.reopen(File::NULL)
+    $stderr.sync = true
+
+    yield
+  ensure
+    $stderr.reopen(old_stderr)
+    old_stderr.close
+  end
 end
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
index bfca4bfb3f7..e7634d2ed4f 100644
--- a/lib/tasks/gitlab/artifacts/migrate.rake
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -15,7 +15,7 @@ namespace :gitlab do
           build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
           build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
 
-          logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
+          logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage")
         rescue => e
           logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
         end
diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake
index 83dd870fa31..c0d6cc8ca8e 100644
--- a/lib/tasks/gitlab/bulk_add_permission.rake
+++ b/lib/tasks/gitlab/bulk_add_permission.rake
@@ -1,6 +1,6 @@
 namespace :gitlab do
   namespace :import do
-    desc "GitLab | Add all users to all projects (admin users are added as masters)"
+    desc "GitLab | Add all users to all projects (admin users are added as maintainers)"
     task all_users_to_all_projects: :environment  do |t, args|
       user_ids = User.where(admin: false).pluck(:id)
       admin_ids = User.where(admin: true).pluck(:id)
@@ -10,11 +10,11 @@ namespace :gitlab do
       ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER)
 
       puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects"
-      ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER)
+      ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER)
     end
 
     desc "GitLab | Add a specific user to all projects (as a developer)"
-    task :user_to_projects, [:email] => :environment  do |t, args|
+    task :user_to_projects, [:email] => :environment do |t, args|
       user = User.find_by(email: args.email)
       project_ids = Project.pluck(:id)
       puts "Importing #{user.email} users into #{project_ids.size} projects"
@@ -22,7 +22,7 @@ namespace :gitlab do
     end
 
     desc "GitLab | Add all users to all groups (admin users are added as owners)"
-    task all_users_to_all_groups: :environment  do |t, args|
+    task all_users_to_all_groups: :environment do |t, args|
       user_ids = User.where(admin: false).pluck(:id)
       admin_ids = User.where(admin: true).pluck(:id)
       groups = Group.all
@@ -36,7 +36,7 @@ namespace :gitlab do
     end
 
     desc "GitLab | Add a specific user to all groups (as a developer)"
-    task :user_to_groups, [:email] => :environment  do |t, args|
+    task :user_to_groups, [:email] => :environment do |t, args|
       user = User.find_by_email args.email
       groups = Group.all
       puts "Importing #{user.email} users into #{groups.size} groups"
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index c04dae7446f..b594f150c3b 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -1,454 +1,66 @@
 namespace :gitlab do
   desc 'GitLab | Check the configuration of GitLab and its environment'
-  task check: %w{gitlab:gitlab_shell:check
-                 gitlab:sidekiq:check
-                 gitlab:incoming_email:check
-                 gitlab:ldap:check
-                 gitlab:app:check}
+  task check: :gitlab_environment do
+    SystemCheck::RakeTask::GitlabTask.run!
+  end
 
   namespace :app do
     desc 'GitLab | Check the configuration of the GitLab Rails app'
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-
-      checks = [
-        SystemCheck::App::GitConfigCheck,
-        SystemCheck::App::DatabaseConfigExistsCheck,
-        SystemCheck::App::MigrationsAreUpCheck,
-        SystemCheck::App::OrphanedGroupMembersCheck,
-        SystemCheck::App::GitlabConfigExistsCheck,
-        SystemCheck::App::GitlabConfigUpToDateCheck,
-        SystemCheck::App::LogWritableCheck,
-        SystemCheck::App::TmpWritableCheck,
-        SystemCheck::App::UploadsDirectoryExistsCheck,
-        SystemCheck::App::UploadsPathPermissionCheck,
-        SystemCheck::App::UploadsPathTmpPermissionCheck,
-        SystemCheck::App::InitScriptExistsCheck,
-        SystemCheck::App::InitScriptUpToDateCheck,
-        SystemCheck::App::ProjectsHaveNamespaceCheck,
-        SystemCheck::App::RedisVersionCheck,
-        SystemCheck::App::RubyVersionCheck,
-        SystemCheck::App::GitVersionCheck,
-        SystemCheck::App::GitUserDefaultSSHConfigCheck,
-        SystemCheck::App::ActiveUsersCheck
-      ]
-
-      SystemCheck.run('GitLab', checks)
+      SystemCheck::RakeTask::AppTask.run!
     end
   end
 
   namespace :gitlab_shell do
     desc "GitLab | Check the configuration of GitLab Shell"
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-      start_checking "GitLab Shell"
-
-      check_gitlab_shell
-      check_repo_base_exists
-      check_repo_base_is_not_symlink
-      check_repo_base_user_and_group
-      check_repo_base_permissions
-      check_repos_hooks_directory_is_link
-      check_gitlab_shell_self_test
-
-      finished_checking "GitLab Shell"
-    end
-
-    # Checks
-    ########################
-
-    def check_repo_base_exists
-      puts "Repo base directory exists?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        if File.exist?(repo_base_path)
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          puts "#{repo_base_path} is missing".color(:red)
-          try_fixing_it(
-            "This should have been created when setting up GitLab Shell.",
-            "Make sure it's set correctly in config/gitlab.yml",
-            "Make sure GitLab Shell is installed correctly."
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repo_base_is_not_symlink
-      puts "Repo storage directories are symlinks?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        unless File.exist?(repo_base_path)
-          puts "can't check because of previous errors".color(:magenta)
-          break
-        end
-
-        unless File.symlink?(repo_base_path)
-          puts "no".color(:green)
-        else
-          puts "yes".color(:red)
-          try_fixing_it(
-            "Make sure it's set to the real directory in config/gitlab.yml"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repo_base_permissions
-      puts "Repo paths access is drwxrws---?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        unless File.exist?(repo_base_path)
-          puts "can't check because of previous errors".color(:magenta)
-          break
-        end
-
-        if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          try_fixing_it(
-            "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
-            "sudo chmod -R ug-s #{repo_base_path}",
-            "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repo_base_user_and_group
-      gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
-      puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
-
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage.legacy_disk_path
-        print "#{name}... "
-
-        unless File.exist?(repo_base_path)
-          puts "can't check because of previous errors".color(:magenta)
-          break
-        end
-
-        user_id = uid_for(gitlab_shell_ssh_user)
-        root_group_id = gid_for('root')
-        group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)]
-        if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid)
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          puts "  User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue)
-          try_fixing_it(
-            "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}"
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_repos_hooks_directory_is_link
-      print "hooks directories in repos are links: ... "
-
-      gitlab_shell_hooks_path = Gitlab.config.gitlab_shell.hooks_path
-
-      unless Project.count > 0
-        puts "can't check, you have no projects".color(:magenta)
-        return
-      end
-
-      puts ""
-
-      Project.find_each(batch_size: 100) do |project|
-        print sanitized_message(project)
-        project_hook_directory = File.join(project.repository.path_to_repo, "hooks")
-
-        if project.empty_repo?
-          puts "repository is empty".color(:magenta)
-        elsif File.directory?(project_hook_directory) && File.directory?(gitlab_shell_hooks_path) &&
-            (File.realpath(project_hook_directory) == File.realpath(gitlab_shell_hooks_path))
-          puts 'ok'.color(:green)
-        else
-          puts "wrong or missing hooks".color(:red)
-          try_fixing_it(
-            sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"),
-            'Check the hooks_path in config/gitlab.yml',
-            'Check your gitlab-shell installation'
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab Shell"
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
-    def check_gitlab_shell_self_test
-      gitlab_shell_repo_base = gitlab_shell_path
-      check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base)
-      puts "Running #{check_cmd}"
-
-      if system(check_cmd, chdir: gitlab_shell_repo_base)
-        puts 'gitlab-shell self-check successful'.color(:green)
-      else
-        puts 'gitlab-shell self-check failed'.color(:red)
-        try_fixing_it(
-          'Make sure GitLab is running;',
-          'Check the gitlab-shell configuration file:',
-          sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}")
-        )
-        fix_and_rerun
-      end
-    end
-
-    # Helper methods
-    ########################
-
-    def gitlab_shell_path
-      Gitlab.config.gitlab_shell.path
-    end
-
-    def gitlab_shell_version
-      Gitlab::Shell.new.version
-    end
-
-    def gitlab_shell_major_version
-      Gitlab::Shell.version_required.split('.')[0].to_i
-    end
-
-    def gitlab_shell_minor_version
-      Gitlab::Shell.version_required.split('.')[1].to_i
+      SystemCheck::RakeTask::GitlabShellTask.run!
     end
+  end
 
-    def gitlab_shell_patch_version
-      Gitlab::Shell.version_required.split('.')[2].to_i
+  namespace :gitaly do
+    desc 'GitLab | Check the health of Gitaly'
+    task check: :gitlab_environment do
+      SystemCheck::RakeTask::GitalyTask.run!
     end
   end
 
   namespace :sidekiq do
     desc "GitLab | Check the configuration of Sidekiq"
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-      start_checking "Sidekiq"
-
-      check_sidekiq_running
-      only_one_sidekiq_running
-
-      finished_checking "Sidekiq"
-    end
-
-    # Checks
-    ########################
-
-    def check_sidekiq_running
-      print "Running? ... "
-
-      if sidekiq_process_count > 0
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          sudo_gitlab("RAILS_ENV=production bin/background_jobs start")
-        )
-        for_more_information(
-          see_installation_guide_section("Install Init Script"),
-          "see log/sidekiq.log for possible errors"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def only_one_sidekiq_running
-      process_count = sidekiq_process_count
-      return if process_count.zero?
-
-      print 'Number of Sidekiq processes ... '
-
-      if process_count == 1
-        puts '1'.color(:green)
-      else
-        puts "#{process_count}".color(:red)
-        try_fixing_it(
-          'sudo service gitlab stop',
-          "sudo pkill -u #{gitlab_user} -f sidekiq",
-          "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq",
-          'sudo service gitlab start'
-        )
-        fix_and_rerun
-      end
-    end
-
-    def sidekiq_process_count
-      ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
-      ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count
+      SystemCheck::RakeTask::SidekiqTask.run!
     end
   end
 
   namespace :incoming_email do
     desc "GitLab | Check the configuration of Reply by email"
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-
-      if Gitlab.config.incoming_email.enabled
-        checks = [
-          SystemCheck::IncomingEmail::ImapAuthenticationCheck
-        ]
-
-        if Rails.env.production?
-          checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
-          checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
-        else
-          checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck
-        end
-
-        SystemCheck.run('Reply by email', checks)
-      else
-        puts 'Reply by email is disabled in config/gitlab.yml'
-      end
+      SystemCheck::RakeTask::IncomingEmailTask.run!
     end
   end
 
   namespace :ldap do
     task :check, [:limit] => :gitlab_environment do |_, args|
-      # Only show up to 100 results because LDAP directories can be very big.
-      # This setting only affects the `rake gitlab:check` script.
-      args.with_defaults(limit: 100)
-      warn_user_is_not_gitlab
-      start_checking "LDAP"
-
-      if Gitlab::Auth::LDAP::Config.enabled?
-        check_ldap(args.limit)
-      else
-        puts 'LDAP is disabled in config/gitlab.yml'
-      end
-
-      finished_checking "LDAP"
-    end
-
-    def check_ldap(limit)
-      servers = Gitlab::Auth::LDAP::Config.providers
-
-      servers.each do |server|
-        puts "Server: #{server}"
+      ENV['LDAP_CHECK_LIMIT'] = args.limit if args.limit.present?
 
-        begin
-          Gitlab::Auth::LDAP::Adapter.open(server) do |adapter|
-            check_ldap_auth(adapter)
-
-            puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"
-
-            users = adapter.users(adapter.config.uid, '*', limit)
-            users.each do |user|
-              puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
-            end
-          end
-        rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e
-          puts "Could not connect to the LDAP server: #{e.message}".color(:red)
-        end
-      end
-    end
-
-    def check_ldap_auth(adapter)
-      auth = adapter.config.has_auth?
-
-      message = if auth && adapter.ldap.bind
-                  'Success'.color(:green)
-                elsif auth
-                  'Failed. Check `bind_dn` and `password` configuration values'.color(:red)
-                else
-                  'Anonymous. No `bind_dn` or `password` configured'.color(:yellow)
-                end
-
-      puts "LDAP authentication... #{message}"
-    end
-  end
-
-  namespace :repo do
-    desc "GitLab | Check the integrity of the repositories managed by GitLab"
-    task check: :gitlab_environment do
-      puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
-      Rake::Task["gitlab:git:fsck"].execute
+      SystemCheck::RakeTask::LdapTask.run!
     end
   end
 
   namespace :orphans do
     desc 'Gitlab | Check for orphaned namespaces and repositories'
     task check: :gitlab_environment do
-      warn_user_is_not_gitlab
-      checks = [
-        SystemCheck::Orphans::NamespaceCheck,
-        SystemCheck::Orphans::RepositoryCheck
-      ]
-
-      SystemCheck.run('Orphans', checks)
+      SystemCheck::RakeTask::OrphansTask.run!
     end
 
     desc 'GitLab | Check for orphaned namespaces in the repositories path'
     task check_namespaces: :gitlab_environment do
-      warn_user_is_not_gitlab
-      checks = [SystemCheck::Orphans::NamespaceCheck]
-
-      SystemCheck.run('Orphans', checks)
+      SystemCheck::RakeTask::Orphans::NamespaceTask.run!
     end
 
     desc 'GitLab | Check for orphaned repositories in the repositories path'
     task check_repositories: :gitlab_environment do
-      warn_user_is_not_gitlab
-      checks = [SystemCheck::Orphans::RepositoryCheck]
-
-      SystemCheck.run('Orphans', checks)
-    end
-  end
-
-  namespace :user do
-    desc "GitLab | Check the integrity of a specific user's repositories"
-    task :check_repos, [:username] => :gitlab_environment do |t, args|
-      username = args[:username] || prompt("Check repository integrity for username? ".color(:blue))
-      user = User.find_by(username: username)
-      if user
-        repo_dirs = user.authorized_projects.map do |p|
-          p.repository.path_to_repo
-        end
-
-        repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
-      else
-        puts "\nUser '#{username}' not found".color(:red)
-      end
-    end
-  end
-
-  # Helper methods
-  ##########################
-
-  def check_gitlab_shell
-    required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version)
-    current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
-
-    print "GitLab Shell version >= #{required_version} ? ... "
-    if current_version.valid? && required_version <= current_version
-      puts "OK (#{current_version})".color(:green)
-    else
-      puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
+      SystemCheck::RakeTask::Orphans::RepositoryTask.run!
     end
   end
 end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index d6d15285489..451ba651674 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -1,41 +1,29 @@
-# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/954
-#
+# frozen_string_literal: true
+require 'set'
+
 namespace :gitlab do
   namespace :cleanup do
-    HASHED_REPOSITORY_NAME = '@hashed'.freeze
-
     desc "GitLab | Cleanup | Clean namespaces"
     task dirs: :gitlab_environment do
-      warn_user_is_not_gitlab
-      remove_flag = ENV['REMOVE']
+      namespaces = Set.new(Namespace.pluck(:path))
+      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
 
-      namespaces  = Namespace.pluck(:path)
-      namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        git_base_path = repository_storage.legacy_disk_path
-        all_dirs = Dir.glob(git_base_path + '/*')
+      Gitaly::Server.all.each do |server|
+        all_dirs = Gitlab::GitalyClient::StorageService
+          .new(server.storage)
+          .list_directories(depth: 0)
+          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
 
-        puts git_base_path.color(:yellow)
         puts "Looking for directories to remove... "
-
-        all_dirs.reject! do |dir|
-          # skip if git repo
-          dir =~ /.git$/
-        end
-
-        all_dirs.reject! do |dir|
-          dir_name = File.basename dir
-
-          # skip if namespace present
-          namespaces.include?(dir_name)
-        end
-
         all_dirs.each do |dir_path|
-          if remove_flag
-            if FileUtils.rm_rf dir_path
-              puts "Removed...#{dir_path}".color(:red)
-            else
-              puts "Cannot remove #{dir_path}".color(:red)
+          if remove?
+            begin
+              Gitlab::GitalyClient::NamespaceService.new(server.storage)
+                .remove(dir_path)
+
+              puts "Removed...#{dir_path}"
+            rescue StandardError => e
+              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
             end
           else
             puts "Can be removed: #{dir_path}".color(:red)
@@ -43,35 +31,36 @@ namespace :gitlab do
         end
       end
 
-      unless remove_flag
+      unless remove?
         puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
       end
     end
 
     desc "GitLab | Cleanup | Clean repositories"
     task repos: :gitlab_environment do
-      warn_user_is_not_gitlab
-
       move_suffix = "+orphaned+#{Time.now.to_i}"
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_root = repository_storage.legacy_disk_path
-        # Look for global repos (legacy, depth 1) and normal repos (depth 2)
-        IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
-          find.each_line do |path|
-            path.chomp!
-            repo_with_namespace = path
-              .sub(repo_root, '')
-              .sub(%r{^/*}, '')
-              .chomp('.git')
-              .chomp('.wiki')
-
-            # TODO ignoring hashed repositories for now.  But revisit to fully support
-            # possible orphaned hashed repos
-            next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace)
-
-            new_path = path + move_suffix
-            puts path.inspect + ' -> ' + new_path.inspect
-            File.rename(path, new_path)
+
+      Gitaly::Server.all.each do |server|
+        Gitlab::GitalyClient::StorageService
+          .new(server.storage)
+          .list_directories
+          .each do |path|
+          repo_with_namespace = path.chomp('.git').chomp('.wiki')
+
+          # TODO ignoring hashed repositories for now.  But revisit to fully support
+          # possible orphaned hashed repos
+          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
+          next if Project.find_by_full_path(repo_with_namespace)
+
+          new_path = path + move_suffix
+          puts path.inspect + ' -> ' + new_path.inspect
+
+          begin
+            Gitlab::GitalyClient::NamespaceService
+              .new(server.storage)
+              .rename(path, new_path)
+          rescue StandardError => e
+            puts "Error occured while moving the repository: #{e.message}".color(:red)
           end
         end
       end
@@ -104,27 +93,46 @@ namespace :gitlab do
       end
     end
 
-    # This is a rake task which removes faulty refs. These refs where only
-    # created in the 8.13.RC cycle, and fixed in the stable builds which were
-    # released. So likely this should only be run once on gitlab.com
-    # Faulty refs are moved so they are kept around, else some features break.
-    desc 'GitLab | Cleanup | Remove faulty deployment refs'
-    task move_faulty_deployment_refs: :gitlab_environment do
-      projects = Project.where(id: Deployment.select(:project_id).distinct)
+    desc "GitLab | Cleanup | Clean orphaned project uploads"
+    task project_uploads: :gitlab_environment do
+      warn_user_is_not_gitlab
 
-      projects.find_each do |project|
-        rugged = project.repository.rugged
+      cleaner = Gitlab::Cleanup::ProjectUploads.new(logger: logger)
+      cleaner.run!(dry_run: dry_run?)
 
-        max_iid = project.deployments.maximum(:iid)
+      if dry_run?
+        logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow)
+      end
+    end
 
-        rugged.references.each('refs/environments/**/*') do |ref|
-          id = ref.name.split('/').last.to_i
-          next unless id > max_iid
+    desc 'GitLab | Cleanup | Clean orphan remote upload files that do not exist in the db'
+    task remote_upload_files: :environment do
+      cleaner = Gitlab::Cleanup::RemoteUploads.new(logger: logger)
+      cleaner.run!(dry_run: dry_run?)
 
-          project.deployments.find(id).create_ref
-          project.repository.delete_refs(ref)
-        end
+      if dry_run?
+        logger.info "To cleanup these files run this command with DRY_RUN=false".color(:yellow)
       end
     end
+
+    def remove?
+      ENV['REMOVE'] == 'true'
+    end
+
+    def dry_run?
+      ENV['DRY_RUN'] != 'false'
+    end
+
+    def logger
+      return @logger if defined?(@logger)
+
+      @logger = if Rails.env.development? || Rails.env.production?
+                  Logger.new(STDOUT).tap do |stdout_logger|
+                    stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger))
+                  end
+                else
+                  Rails.logger
+                end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 139ab70e125..74cd70c6e9f 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -46,9 +46,13 @@ namespace :gitlab do
 
     desc 'Configures the database by running migrate, or by loading the schema and seeding if needed'
     task configure: :environment do
-      if ActiveRecord::Base.connection.tables.any?
+      # Check if we have existing db tables
+      # The schema_migrations table will still exist if drop_tables was called
+      if ActiveRecord::Base.connection.tables.count > 1
         Rake::Task['db:migrate'].invoke
       else
+        # Add post-migrate paths to ensure we mark all migrations as up
+        Gitlab::Database.add_post_migrate_path_to_rails(force: true)
         Rake::Task['db:schema:load'].invoke
         Rake::Task['db:seed_fu'].invoke
       end
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index cb4f7e5c8a8..8a53b51d4fe 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -1,87 +1,24 @@
 namespace :gitlab do
   namespace :git do
-    desc "GitLab | Git | Repack"
-    task repack: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    desc "GitLab | Git | Run garbage collection on all repos"
-    task gc: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    desc "GitLab | Git | Prune all repos"
-    task prune: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune")
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
     desc 'GitLab | Git | Check all repos integrity'
     task fsck: :gitlab_environment do
-      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
-        check_config_lock(repo)
-        check_ref_locks(repo)
-      end
-
-      if failures.empty?
-        puts "Done".color(:green)
-      else
-        output_failures(failures)
-      end
-    end
-
-    def perform_git_cmd(cmd, message)
-      puts "Starting #{message} on all repositories"
-
       failures = []
-      all_repos do |repo|
-        if system(*cmd, chdir: repo)
-          puts "Performed #{message} at #{repo}"
-        else
-          failures << repo
+      Project.find_each(batch_size: 100) do |project|
+        begin
+          project.repository.fsck
+
+        rescue => e
+          failures << "#{project.full_path} on #{project.repository_storage}: #{e}"
         end
 
-        yield(repo) if block_given?
+        puts "Performed integrity check for #{project.repository.full_path}"
       end
 
-      failures
-    end
-
-    def output_failures(failures)
-      puts "The following repositories reported errors:".color(:red)
-      failures.each { |f| puts "- #{f}" }
-    end
-
-    def check_config_lock(repo_dir)
-      config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
-      config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
-
-      puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
-    end
-
-    def check_ref_locks(repo_dir)
-      lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
-
-      if lock_files.present?
-        puts "Ref lock files exist:".color(:red)
-
-        lock_files.each { |lock_file| puts "  #{lock_file}" }
+      if failures.empty?
+        puts "Done".color(:green)
       else
-        puts "No ref lock files exist".color(:green)
+        puts "The following repositories reported errors:".color(:red)
+        failures.each { |f| puts "- #{f}" }
       end
     end
   end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index e9ca6404fe8..80de3d2ef51 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -1,13 +1,12 @@
 namespace :gitlab do
   namespace :gitaly do
     desc "GitLab | Install or upgrade gitaly"
-    task :install, [:dir, :repo] => :gitlab_environment do |t, args|
-      require 'toml-rb'
-
+    task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
 
-      unless args.dir.present?
-        abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
+      unless args.dir.present? && args.storage_path.present?
+        abort %(Please specify the directory where you want to install gitaly and the path for the default storage
+Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       end
 
       args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
@@ -27,7 +26,8 @@ namespace :gitlab do
           "BUNDLE_PATH=#{Bundler.bundle_path}")
       end
 
-      Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
+      storage_paths = { 'default' => args.storage_path }
+      Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths)
       Dir.chdir(args.dir) do
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
@@ -35,17 +35,5 @@ namespace :gitlab do
         end
       end
     end
-
-    desc "GitLab | Print storage configuration in TOML format"
-    task storage_config: :environment do
-      require 'toml-rb'
-
-      puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
-      puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
-
-      # Exclude gitaly-ruby configuration because that depends on the gitaly
-      # installation directory.
-      puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
-    end
   end
 end
diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake
index 44074397c05..900dbf7be24 100644
--- a/lib/tasks/gitlab/import_export.rake
+++ b/lib/tasks/gitlab/import_export.rake
@@ -10,15 +10,22 @@ namespace :gitlab do
       puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
     end
 
-    desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz'
-    task bump_test_version: :environment do
-      Dir.mktmpdir do |tmp_dir|
-        system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null")
-        File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
-        system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null")
+    desc 'GitLab | Bumps the Import/Export version in fixtures and project templates'
+    task bump_version: :environment do
+      archives = Dir['vendor/project_templates/*.tar.gz']
+      archives.push('spec/features/projects/import_export/test_project_export.tar.gz')
+
+      archives.each do |archive|
+        raise ArgumentError unless File.exist?(archive)
+
+        Dir.mktmpdir do |tmp_dir|
+          system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null")
+          File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
+          system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null")
+        end
       end
 
-      puts "Updated to #{Gitlab::ImportExport.version}"
+      puts "Updated #{archives} to #{Gitlab::ImportExport.version}."
     end
   end
 end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 289aa5d9060..e97d77d20e0 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -54,8 +54,8 @@ namespace :gitlab do
       puts "HTTP Clone URL:\t#{http_clone_url}"
       puts "SSH Clone URL:\t#{ssh_clone_url}"
       puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".color(:green) : "no"}"
-      puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}"
-      puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled
+      puts "Using Omniauth:\t#{Gitlab::Auth.omniauth_enabled? ? "yes".color(:green) : "no"}"
+      puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled?
 
       # check Gitolite version
       gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION"
@@ -67,8 +67,10 @@ namespace :gitlab do
       puts "GitLab Shell".color(:yellow)
       puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
       puts "Repository storage paths:"
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        puts "- #{name}: \t#{repository_storage.legacy_disk_path}"
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        Gitlab.config.repositories.storages.each do |name, repository_storage|
+          puts "- #{name}: \t#{repository_storage.legacy_disk_path}"
+        end
       end
       puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
       puts "Git:\t\t#{Gitlab.config.git.bin_path}"
diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake
index c66a2a263dc..0459de27c96 100644
--- a/lib/tasks/gitlab/ldap.rake
+++ b/lib/tasks/gitlab/ldap.rake
@@ -1,7 +1,7 @@
 namespace :gitlab do
   namespace :ldap do
     desc 'GitLab | LDAP | Rename provider'
-    task :rename_provider, [:old_provider, :new_provider] => :environment  do |_, args|
+    task :rename_provider, [:old_provider, :new_provider] => :gitlab_environment do |_, args|
       old_provider = args[:old_provider] ||
         prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue))
       new_provider = args[:new_provider] ||
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 4fcbbbf8c9d..0ebc6f00793 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -92,9 +92,11 @@ namespace :gitlab do
   def setup
     warn_user_is_not_gitlab
 
+    ensure_write_to_authorized_keys_is_enabled
+
     unless ENV['force'] == 'yes'
-      puts "This will rebuild an authorized_keys file."
-      puts "You will lose any data stored in authorized_keys file."
+      puts "This task will now rebuild the authorized_keys file."
+      puts "You will lose any data stored in the authorized_keys file."
       ask_to_continue
       puts ""
     end
@@ -118,4 +120,44 @@ namespace :gitlab do
     puts "Quitting...".color(:red)
     exit 1
   end
+
+  def ensure_write_to_authorized_keys_is_enabled
+    return if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
+
+    puts authorized_keys_is_disabled_warning
+
+    unless ENV['force'] == 'yes'
+      puts 'Do you want to permanently enable the "Write to authorized_keys file" setting now?'
+      ask_to_continue
+    end
+
+    puts 'Enabling the "Write to authorized_keys file" setting...'
+    Gitlab::CurrentSettings.current_application_settings.update!(authorized_keys_enabled: true)
+
+    puts 'Successfully enabled "Write to authorized_keys file"!'
+    puts ''
+  end
+
+  def authorized_keys_is_disabled_warning
+    <<-MSG.strip_heredoc
+      WARNING
+
+      The "Write to authorized_keys file" setting is disabled, which prevents
+      the file from being rebuilt!
+
+      It should be enabled for most GitLab installations. Large installations
+      may wish to disable it as part of speeding up SSH operations.
+
+      See https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.html
+
+      If you did not intentionally disable this option in Admin Area > Settings,
+      then you may have been affected by the 9.3.0 bug in which the new setting
+      was disabled by default.
+
+      https://gitlab.com/gitlab-org/gitlab-ee/issues/2738
+
+      It was reverted in 9.3.1 and fixed in 9.3.3, however, if Settings were
+      saved while the setting was unchecked, then it is still disabled.
+    MSG
+  end
 end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index 68d6f9d7cb1..f9ce3e1d338 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -2,11 +2,34 @@ namespace :gitlab do
   namespace :storage do
     desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
     task migrate_to_hashed: :environment do
-      legacy_projects_count = Project.with_unmigrated_storage.count
+      if Gitlab::Database.read_only?
+        warn 'This task requires database write access. Exiting.'
+
+        next
+      end
+
+      storage_migrator = Gitlab::HashedStorage::Migrator.new
       helper = Gitlab::HashedStorage::RakeHelper
 
+      if helper.range_single_item?
+        project = Project.with_unmigrated_storage.find_by(id: helper.range_from)
+
+        unless project
+          warn "There are no projects requiring storage migration with ID=#{helper.range_from}"
+
+          next
+        end
+
+        puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..."
+        storage_migrator.migrate(project)
+
+        next
+      end
+
+      legacy_projects_count = Project.with_unmigrated_storage.count
+
       if legacy_projects_count == 0
-        puts 'There are no projects requiring storage migration. Nothing to do!'
+        warn 'There are no projects requiring storage migration. Nothing to do!'
 
         next
       end
@@ -14,7 +37,7 @@ namespace :gitlab do
       print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}"
 
       helper.project_id_batches do |start, finish|
-        StorageMigratorWorker.perform_async(start, finish)
+        storage_migrator.bulk_schedule(start: start, finish: finish)
 
         print '.'
       end
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
index fd2a4f2d11a..5a232091a7e 100644
--- a/lib/tasks/gitlab/traces.rake
+++ b/lib/tasks/gitlab/traces.rake
@@ -8,9 +8,7 @@ namespace :gitlab do
       logger = Logger.new(STDOUT)
       logger.info('Archiving legacy traces')
 
-      Ci::Build.finished
-        .where('NOT EXISTS (?)',
-          Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+      Ci::Build.finished.without_archived_trace
         .order(id: :asc)
         .find_in_batches(batch_size: 1000) do |jobs|
         job_ids = jobs.map { |job| [job.id] }
@@ -20,5 +18,22 @@ namespace :gitlab do
         logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}")
       end
     end
+
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of job traces')
+
+      Ci::Build.joins(:project)
+        .with_archived_trace_stored_locally
+        .find_each(batch_size: 10) do |build|
+        begin
+          build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE)
+
+          logger.info("Transferred job trace of #{build.id} to object storage")
+        rescue => e
+          logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+        end
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index a25f7ce59c7..abe10f5580e 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -6,6 +6,8 @@ namespace :gitlab do
 
   desc "GitLab | Update project templates"
   task :update_project_templates do
+    include Gitlab::ImportExport::CommandLineUtil
+
     if Rails.env.production?
       puts "This rake task is not meant fo production instances".red
       exit(1)
@@ -52,7 +54,7 @@ namespace :gitlab do
       end
 
       Projects::ImportExport::ExportService.new(project, admin).execute
-      FileUtils.cp(project.export_project_path, template.archive_path)
+      download_or_copy_upload(project.export_file, template.archive_path)
       Projects::DestroyService.new(admin, project).execute
       puts "Exported #{template.name}".green
     end
@@ -98,10 +100,6 @@ namespace :gitlab do
       /(\.{1,2}|LICENSE|Global|\.gitignore)\z/
     ),
     Template.new(
-      "https://gitlab.com/gitlab-org/gitlab-ci-yml.git",
-      /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/
-    ),
-    Template.new(
       "https://gitlab.com/gitlab-org/Dockerfile.git",
       /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/
     )
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
index 78e18992a8e..1c93609a006 100644
--- a/lib/tasks/gitlab/uploads/migrate.rake
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -1,6 +1,30 @@
 namespace :gitlab do
   namespace :uploads do
-    desc 'GitLab | Uploads | Migrate the uploaded files to object storage'
+    namespace :migrate do
+      desc "GitLab | Uploads | Migrate all uploaded files to object storage"
+      task all: :environment do
+        categories = [%w(AvatarUploader Project :avatar),
+                      %w(AvatarUploader Group :avatar),
+                      %w(AvatarUploader User :avatar),
+                      %w(AttachmentUploader Note :attachment),
+                      %w(AttachmentUploader Appearance :logo),
+                      %w(AttachmentUploader Appearance :header_logo),
+                      %w(FaviconUploader Appearance :favicon),
+                      %w(FileUploader Project),
+                      %w(PersonalFileUploader Snippet),
+                      %w(NamespaceFileUploader Snippet),
+                      %w(FileUploader MergeRequest)]
+
+        categories.each do |args|
+          Rake::Task["gitlab:uploads:migrate"].invoke(*args)
+          Rake::Task["gitlab:uploads:migrate"].reenable
+        end
+      end
+    end
+
+    # The following is the actual rake task that migrates uploads of specified
+    # category to object storage
+    desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage'
     task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
       batch_size     = ENV.fetch('BATCH', 200).to_i
       @to_store      = ObjectStorage::Store::REMOTE
@@ -8,7 +32,7 @@ namespace :gitlab do
       @uploader_class = args.uploader_class.constantize
       @model_class    = args.model_class.constantize
 
-      uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches
+      uploads.each_batch(of: batch_size, &method(:enqueue_batch))
     end
 
     def enqueue_batch(batch, index)
diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake
index 5a1c8006052..15cec80b6a6 100644
--- a/lib/tasks/gitlab/web_hook.rake
+++ b/lib/tasks/gitlab/web_hook.rake
@@ -25,11 +25,22 @@ namespace :gitlab do
       web_hook_url = ENV['URL']
       namespace_path = ENV['NAMESPACE']
 
-      projects = find_projects(namespace_path)
-      project_ids = projects.pluck(:id)
+      web_hooks = find_web_hooks(namespace_path)
 
       puts "Removing webhooks with the url '#{web_hook_url}' ... "
-      count = WebHook.where(url: web_hook_url, project_id: project_ids, type: 'ProjectHook').delete_all
+
+      # FIXME: Hook URLs are now encrypted, so there is no way to efficiently
+      # find them all in SQL. For now, check them in Ruby. If this is too slow,
+      # we could consider storing a hash of the URL alongside the encrypted
+      # value to speed up searches
+      count = 0
+      web_hooks.find_each do |hook|
+        next unless hook.url == web_hook_url
+
+        hook.destroy!
+        count += 1
+      end
+
       puts "#{count} webhooks were removed."
     end
 
@@ -37,29 +48,37 @@ namespace :gitlab do
     task list: :environment do
       namespace_path = ENV['NAMESPACE']
 
-      projects = find_projects(namespace_path)
-      web_hooks = projects.all.map(&:hooks).flatten
-      web_hooks.each do |hook|
+      web_hooks = find_web_hooks(namespace_path)
+      web_hooks.find_each do |hook|
         puts "#{hook.project.name.truncate(20).ljust(20)} -> #{hook.url}"
       end
 
-      puts "\n#{web_hooks.size} webhooks found."
+      puts "\n#{web_hooks.count} webhooks found."
     end
   end
 
   def find_projects(namespace_path)
     if namespace_path.blank?
       Project
-    elsif namespace_path == '/'
-      Project.in_namespace(nil)
     else
-      namespace = Namespace.where(path: namespace_path).first
-      if namespace
-        Project.in_namespace(namespace.id)
-      else
+      namespace = Namespace.find_by_full_path(namespace_path)
+
+      unless namespace
         puts "Namespace not found: #{namespace_path}".color(:red)
         exit 2
       end
+
+      Project.in_namespace(namespace.id)
+    end
+  end
+
+  def find_web_hooks(namespace_path)
+    if namespace_path.blank?
+      ProjectHook
+    else
+      project_ids = find_projects(namespace_path).select(:id)
+
+      ProjectHook.where(project_id: project_ids)
     end
   end
 end
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
index ad2d034b0b4..786efd14b1a 100644
--- a/lib/tasks/haml-lint.rake
+++ b/lib/tasks/haml-lint.rake
@@ -2,5 +2,16 @@ unless Rails.env.production?
   require 'haml_lint/rake_task'
   require 'haml_lint/inline_javascript'
 
+  # Workaround for warnings from parser/current
+  # Keep it even if it no longer emits any warnings,
+  # because we'll still see warnings in console/server anyway,
+  # and we don't need to break static-analysis for this.
+  task :haml_lint do
+    require 'parser'
+    def Parser.warn(*args)
+      puts(*args) # static-analysis ignores stdout if status is 0
+    end
+  end
+
   HamlLint::RakeTask.new
 end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index aafbe52e5f8..f912f521dfb 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -9,7 +9,10 @@ class GithubImport
   def initialize(token, gitlab_username, project_path, extras)
     @options = { token: token }
     @project_path = project_path
-    @current_user = User.find_by_username(gitlab_username)
+    @current_user = UserFinder.new(gitlab_username).find_by_username
+
+    raise "GitLab user #{gitlab_username} not found. Please specify a valid username." unless @current_user
+
     @github_repo = extras.empty? ? nil : extras.first
   end
 
@@ -39,7 +42,7 @@ class GithubImport
   end
 
   def import!
-    @project.force_import_start
+    @project.import_state.force_start
 
     import_success = false
 
@@ -50,11 +53,11 @@ class GithubImport
     end
 
     if import_success
-      @project.import_finish
+      @project.after_import
       puts "Import finished. Timings: #{timings}".color(:green)
     else
       puts "Import was not successful. Errors were as follows:"
-      puts @project.import_error
+      puts @project.import_state.last_error
     end
   end
 
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index 8b86a5c72a5..5d673a1a285 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -17,16 +17,25 @@ unless Rails.env.production?
       Rake::Task['eslint'].invoke
     end
 
+    desc "GitLab | lint | Lint HAML files"
+    task :haml do
+      begin
+        Rake::Task['haml_lint'].invoke
+      rescue RuntimeError # The haml_lint tasks raise a RuntimeError
+        exit(1)
+      end
+    end
+
     desc "GitLab | lint | Run several lint checks"
     task :all do
       status = 0
 
       %w[
         config_lint
-        haml_lint
+        lint:haml
         scss_lint
-        flay
         gettext:lint
+        gettext:updated_check
         lint:static_verification
       ].each do |task|
         pid = Process.fork do
@@ -38,13 +47,12 @@ unless Rails.env.production?
           $stderr.reopen(wr_err)
 
           begin
-            begin
-              Rake::Task[task].invoke
-            rescue RuntimeError # The haml_lint tasks raise a RuntimeError
-              exit(1)
-            end
+            Rake::Task[task].invoke
           rescue SystemExit => ex
-            msg = "*** Rake task #{task} failed with the following error(s):"
+            msg = "*** Rake task #{task} exited:"
+            raise ex
+          rescue => ex
+            msg = "*** Rake task #{task} raised #{ex.class}:"
             raise ex
           ensure
             $stdout.reopen(stdout)
diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake
index 9b05876034c..c77fa49d586 100644
--- a/lib/tasks/migrate/add_limits_mysql.rake
+++ b/lib/tasks/migrate/add_limits_mysql.rake
@@ -3,6 +3,7 @@ require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
 require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
 require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
 require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql')
+require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
@@ -12,4 +13,5 @@ task add_limits_mysql: :environment do
   MergeRequestDiffFileLimitsToMysql.new.up
   LimitsCiBuildTraceChunksRawDataForMysql.new.up
   IncreaseMysqlTextLimitForGpgKeys.new.up
+  PrometheusMetricsLimitsToMysql.new.up
 end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index e7aab50e42a..f69d204c579 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -22,3 +22,18 @@ task setup_postgresql: :environment do
   ProjectNameLowerIndex.new.up
   AddPathIndexToRedirectRoutes.new.up
 end
+
+desc 'GitLab | Generate PostgreSQL Password Hash'
+task :postgresql_md5_hash do
+  require 'digest'
+  username = ENV.fetch('USERNAME') do |missing|
+    puts "You must provide an username with '#{missing}' ENV variable"
+    exit(1)
+  end
+  password = ENV.fetch('PASSWORD') do |missing|
+    puts "You must provide a password with '#{missing}' ENV variable"
+    exit(1)
+  end
+  hash = Digest::MD5.hexdigest("#{password}#{username}")
+  puts "The MD5 hash of your database password for user: #{username} -> #{hash}"
+end
diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake
index 81829668de8..eec024f9bbb 100644
--- a/lib/tasks/tokens.rake
+++ b/lib/tasks/tokens.rake
@@ -1,4 +1,7 @@
 require_relative '../../app/models/concerns/token_authenticatable.rb'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/base.rb'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure.rb'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/digest.rb'
 
 namespace :tokens do
   desc "Reset all GitLab incoming email tokens"
@@ -26,13 +29,6 @@ class TmpUser < ActiveRecord::Base
 
   self.table_name = 'users'
 
-  def reset_incoming_email_token!
-    write_new_token(:incoming_email_token)
-    save!(validate: false)
-  end
-
-  def reset_feed_token!
-    write_new_token(:feed_token)
-    save!(validate: false)
-  end
+  add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) }
+  add_authentication_token_field :feed_token
 end