Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (1723 commits)
  Resolve "Editor icons"
  Refactor issuable destroy action
  Ignore routes matching legacy_*_redirect in route specs
  Gitlab::Git::RevList and LfsChanges use lazy popen
  Gitlab::Git::Popen can lazily hand output to a block
  Merge branch 'master-i18n' into 'master'
  Remove unique validation from external_url in Environment
  Expose `duration` in Job API entity
  Add TimeCop freeze for DST and Regular time
  Harcode project visibility
  update a changelog
  Put a condition to old migration that adds fast_forward column to MRs
  Expose project visibility as CI variable
  fix flaky tests by removing unneeded clicks and focus actions
  fix flaky test in gfm_autocomplete_spec.rb
  Use Gitlab::Git operations for repository mirroring
  Encapsulate git operations for mirroring in Gitlab::Git
  Create a Wiki Repository's raw_repository properly
  Add `Gitlab::Git::Repository#fetch` command
  Fix Gitlab::Metrics::System#real_time and #monotonic_time doc
  ...

8 files changed, 119 insertions, 33 deletions

diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index 259a755d724..a42f02a84fd 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -3,8 +3,8 @@ namespace :gitlab do
     desc 'GitLab | Assets | Compile all frontend assets'
     task compile: [
       'yarn:check',
-      'rake:assets:precompile',
       'gettext:po_to_json',
+      'rake:assets:precompile',
       'webpack:compile',
       'fix_urls'
     ]
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
index 3eade7bf553..ba221e44e5d 100644
--- a/lib/tasks/gitlab/dev.rake
+++ b/lib/tasks/gitlab/dev.rake
@@ -4,7 +4,11 @@ namespace :gitlab do
     task :ee_compat_check, [:branch] => :environment do |_, args|
       opts =
         if ENV['CI']
-          { branch: ENV['CI_COMMIT_REF_NAME'] }
+          {
+            ce_project_url: ENV['CI_PROJECT_URL'],
+            branch: ENV['CI_COMMIT_REF_NAME'],
+            job_id: ENV['CI_JOB_ID']
+          }
         else
           unless args[:branch]
             puts "Must specify a branch as an argument".color(:red)
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 08677a98fc1..8377fe3269d 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -50,6 +50,8 @@ namespace :gitlab do
     # only generate a configuration for the most common and simplest case: when
     # we have exactly one Gitaly process and we are sure it is running locally
     # because it uses a Unix socket.
+    # For development and testing purposes, an extra storage is added to gitaly,
+    # which is not known to Rails, but must be explicitly stubbed.
     def gitaly_configuration_toml(gitaly_ruby: true)
       storages = []
       address = nil
@@ -67,6 +69,11 @@ namespace :gitlab do
 
         storages << { name: key, path: val['path'] }
       end
+
+      if Rails.env.test?
+        storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
+      end
+
       config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
       config[:auth] = { token: 'secret' } if Rails.env.test?
       config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 42825f29e32..0e6aed32c52 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -79,7 +79,7 @@ namespace :gitlab do
         if File.exist?(path_to_repo)
           print '-'
         else
-          if Gitlab::Shell.new.add_repository(project.repository_storage_path,
+          if Gitlab::Shell.new.add_repository(project.repository_storage,
                                               project.disk_path)
             print '.'
           else
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
new file mode 100644
index 00000000000..e05be4a3405
--- /dev/null
+++ b/lib/tasks/gitlab/storage.rake
@@ -0,0 +1,85 @@
+namespace :gitlab do
+  namespace :storage do
+    desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
+    task migrate_to_hashed: :environment do
+      legacy_projects_count = Project.with_legacy_storage.count
+
+      if legacy_projects_count == 0
+        puts 'There are no projects using legacy storage. Nothing to do!'
+
+        next
+      end
+
+      print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}"
+
+      project_id_batches do |start, finish|
+        StorageMigratorWorker.perform_async(start, finish)
+
+        print '.'
+      end
+
+      puts ' Done!'
+    end
+
+    desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
+    task legacy_projects: :environment do
+      projects_summary(Project.with_legacy_storage)
+    end
+
+    desc 'Gitlab | Storage | List existing projects using Legacy Storage'
+    task list_legacy_projects: :environment do
+      projects_list(Project.with_legacy_storage)
+    end
+
+    desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
+    task hashed_projects: :environment do
+      projects_summary(Project.with_hashed_storage)
+    end
+
+    desc 'Gitlab | Storage | List existing projects using Hashed Storage'
+    task list_hashed_projects: :environment do
+      projects_list(Project.with_hashed_storage)
+    end
+
+    def batch_size
+      ENV.fetch('BATCH', 200).to_i
+    end
+
+    def project_id_batches(&block)
+      Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+        ids = relation.pluck(:id)
+
+        yield ids.min, ids.max
+      end
+    end
+
+    def projects_summary(relation)
+      projects_count = relation.count
+      puts "* Found #{projects_count} projects".color(:green)
+
+      projects_count
+    end
+
+    def projects_list(relation)
+      projects_count = projects_summary(relation)
+
+      projects = relation.with_route
+      limit = ENV.fetch('LIMIT', 500).to_i
+
+      return unless projects_count > 0
+
+      puts "  ! Displaying first #{limit} projects..." if projects_count > limit
+
+      counter = 0
+      projects.find_in_batches(batch_size: batch_size) do |batch|
+        batch.each do |project|
+          counter += 1
+
+          puts "  - #{project.full_path} (id: #{project.id})".color(:red)
+
+          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake
deleted file mode 100644
index 3a16ace60bd..00000000000
--- a/lib/tasks/gitlab/users.rake
+++ /dev/null
@@ -1,11 +0,0 @@
-namespace :gitlab do
-  namespace :users do
-    desc "GitLab | Clear the authentication token for all users"
-    task clear_all_authentication_tokens: :environment  do |t, args|
-      # Do small batched updates because these updates will be slow and locking
-      User.select(:id).find_in_batches(batch_size: 100) do |batch|
-        User.where(id: batch.map(&:id)).update_all(authentication_token: nil)
-      end
-    end
-  end
-end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 4d485108cf6..7f86fd7b45e 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -39,13 +39,19 @@ class GithubImport
   def import!
     @project.force_import_start
 
+    import_success = false
+
     timings = Benchmark.measure do
-      Github::Import.new(@project, @options).execute
+      import_success = Github::Import.new(@project, @options).execute
     end
 
-    puts "Import finished. Timings: #{timings}".color(:green)
-
-    @project.import_finish
+    if import_success
+      @project.import_finish
+      puts "Import finished. Timings: #{timings}".color(:green)
+    else
+      puts "Import was not successful. Errors were as follows:"
+      puts @project.import_error
+    end
   end
 
   def new_project
@@ -53,18 +59,23 @@ class GithubImport
       namespace_path, _sep, name = @project_path.rpartition('/')
       namespace = find_or_create_namespace(namespace_path)
 
-      Projects::CreateService.new(
+      project = Projects::CreateService.new(
         @current_user,
         name: name,
         path: name,
         description: @repo['description'],
         namespace_id: namespace.id,
         visibility_level: visibility_level,
-        import_type: 'github',
-        import_source: @repo['full_name'],
-        import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"),
         skip_wiki: @repo['has_wiki']
       ).execute
+
+      project.update!(
+        import_type: 'github',
+        import_source: @repo['full_name'],
+        import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@")
+      )
+
+      project
     end
   end
 
diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake
index ad1818ff1fa..693597afdf8 100644
--- a/lib/tasks/tokens.rake
+++ b/lib/tasks/tokens.rake
@@ -1,12 +1,7 @@
 require_relative '../../app/models/concerns/token_authenticatable.rb'
 
 namespace :tokens do
-  desc "Reset all GitLab user auth tokens"
-  task reset_all_auth: :environment do
-    reset_all_users_token(:reset_authentication_token!)
-  end
-
-  desc "Reset all GitLab email tokens"
+  desc "Reset all GitLab incoming email tokens"
   task reset_all_email: :environment do
     reset_all_users_token(:reset_incoming_email_token!)
   end
@@ -31,11 +26,6 @@ class TmpUser < ActiveRecord::Base
 
   self.table_name = 'users'
 
-  def reset_authentication_token!
-    write_new_token(:authentication_token)
-    save!(validate: false)
-  end
-
   def reset_incoming_email_token!
     write_new_token(:incoming_email_token)
     save!(validate: false)