Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (170 commits)
  support ordering of project notes in notes api
  Redirect to an already forked project if it exists
  Reschedule the migration to populate fork networks
  Create fork networks for forks for which the source was deleted.
  Fix item name and namespace text overflow in Projects dropdown
  Minor backport from EE
  fix link that was linking to `html` instead of `md`
  Backport epic tasklist
  Add timeouts for Gitaly calls
  SSHUploadPack over Gitaly is now OptOut
  fix icon colors in commit list
  Fix star icon color/stroke
  Backport border inline edit
  Add checkboxes to automatically run AutoDevops pipeline
  BE for automatic pipeline when enabling Auto DevOps
  I am certainly weary of debugging sidekiq but I don't think that's what was meant
  Ensure MRs always use branch refs for comparison
  Fix issue comment submit button disabled on GFM paste
  Lock seed-fu at the correct version in Gemfile.lock
  Improve indexes on merge_request_diffs
  ...

4 files changed, 78 insertions, 18 deletions

diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake
index 99b3168d9eb..2301ec9b228 100644
--- a/lib/tasks/brakeman.rake
+++ b/lib/tasks/brakeman.rake
@@ -2,7 +2,7 @@ desc 'Security check via brakeman'
 task :brakeman do
   # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge
   # requests are welcome!
-  if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z))
+  if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z))
     puts 'Security check succeed'
   else
     puts 'Security check failed'
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 301affc9522..eb0f757aea7 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -1,11 +1,14 @@
 namespace :gitlab do
   namespace :cleanup do
+    HASHED_REPOSITORY_NAME = '@hashed'.freeze
+
     desc "GitLab | Cleanup | Clean namespaces"
     task dirs: :environment  do
       warn_user_is_not_gitlab
       remove_flag = ENV['REMOVE']
 
-      namespaces = Namespace.pluck(:path)
+      namespaces  = Namespace.pluck(:path)
+      namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
       Gitlab.config.repositories.storages.each do |name, repository_storage|
         git_base_path = repository_storage['path']
         all_dirs = Dir.glob(git_base_path + '/*')
@@ -62,7 +65,7 @@ namespace :gitlab do
 
             # TODO ignoring hashed repositories for now.  But revisit to fully support
             # possible orphaned hashed repos
-            next if repo_with_namespace.start_with?('@hashed/') || Project.find_by_full_path(repo_with_namespace)
+            next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace)
 
             new_path = path + move_suffix
             puts path.inspect + ' -> ' + new_path.inspect
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index f2002d7a426..4d880c05f99 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -78,6 +78,8 @@ namespace :gitlab do
       config[:auth] = { token: 'secret' } if Rails.env.test?
       config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
       config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
+      config[:bin_dir] = Gitlab.config.gitaly.client_path
+
       TOML.dump(config)
     end
 
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index e05be4a3405..8ac73bc8ff2 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -2,10 +2,10 @@ namespace :gitlab do
   namespace :storage do
     desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
     task migrate_to_hashed: :environment do
-      legacy_projects_count = Project.with_legacy_storage.count
+      legacy_projects_count = Project.with_unmigrated_storage.count
 
       if legacy_projects_count == 0
-        puts 'There are no projects using legacy storage. Nothing to do!'
+        puts 'There are no projects requiring storage migration. Nothing to do!'
 
         next
       end
@@ -23,22 +23,42 @@ namespace :gitlab do
 
     desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
     task legacy_projects: :environment do
-      projects_summary(Project.with_legacy_storage)
+      relation_summary('projects', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Legacy Storage'
     task list_legacy_projects: :environment do
-      projects_list(Project.with_legacy_storage)
+      projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
     task hashed_projects: :environment do
-      projects_summary(Project.with_hashed_storage)
+      relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Hashed Storage'
     task list_hashed_projects: :environment do
-      projects_list(Project.with_hashed_storage)
+      projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
+    task legacy_attachments: :environment do
+      relation_summary('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
+    task list_legacy_attachments: :environment do
+      attachments_list('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
+    task hashed_attachments: :environment do
+      relation_summary('attachments using Hashed Storage', hashed_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
+    task list_hashed_attachments: :environment do
+      attachments_list('attachments using Hashed Storage', hashed_attachments_relation)
     end
 
     def batch_size
@@ -46,29 +66,43 @@ namespace :gitlab do
     end
 
     def project_id_batches(&block)
-      Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+      Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
         ids = relation.pluck(:id)
 
         yield ids.min, ids.max
       end
     end
 
-    def projects_summary(relation)
-      projects_count = relation.count
-      puts "* Found #{projects_count} projects".color(:green)
+    def legacy_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+          ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def hashed_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+        ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def relation_summary(relation_name, relation)
+      relation_count = relation.count
+      puts "* Found #{relation_count} #{relation_name}".color(:green)
 
-      projects_count
+      relation_count
     end
 
-    def projects_list(relation)
-      projects_count = projects_summary(relation)
+    def projects_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
 
       projects = relation.with_route
       limit = ENV.fetch('LIMIT', 500).to_i
 
-      return unless projects_count > 0
+      return unless relation_count > 0
 
-      puts "  ! Displaying first #{limit} projects..." if projects_count > limit
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
 
       counter = 0
       projects.find_in_batches(batch_size: batch_size) do |batch|
@@ -81,5 +115,26 @@ namespace :gitlab do
         end
       end
     end
+
+    def attachments_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
+
+      limit = ENV.fetch('LIMIT', 500).to_i
+
+      return unless relation_count > 0
+
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
+
+      counter = 0
+      relation.find_in_batches(batch_size: batch_size) do |batch|
+        batch.each do |upload|
+          counter += 1
+
+          puts "  - #{upload.path} (id: #{upload.id})".color(:red)
+
+          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+    end
   end
 end