Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

7 files changed, 121 insertions, 75 deletions

diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index cc10d73f76a..0bca63a64f5 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -9,14 +9,9 @@ namespace :gitlab do
     task create: :gitlab_environment do
       warn_user_is_not_gitlab
 
-      Rake::Task['gitlab:backup:db:create'].invoke
-      Rake::Task['gitlab:backup:repo:create'].invoke
-      Rake::Task['gitlab:backup:uploads:create'].invoke
-      Rake::Task['gitlab:backup:builds:create'].invoke
-      Rake::Task['gitlab:backup:artifacts:create'].invoke
-      Rake::Task['gitlab:backup:pages:create'].invoke
-      Rake::Task['gitlab:backup:lfs:create'].invoke
-      Rake::Task['gitlab:backup:registry:create'].invoke
+      %w(db repo uploads builds artifacts pages lfs terraform_state registry packages).each do |type|
+        Rake::Task["gitlab:backup:#{type}:create"].invoke
+      end
 
       backup = Backup::Manager.new(progress)
       backup.write_info
@@ -83,7 +78,9 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
       Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages')
       Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
+      Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state')
       Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
+      Rake::Task['gitlab:backup:packages:restore'].invoke unless backup.skipped?('packages')
       Rake::Task['gitlab:shell:setup'].invoke
       Rake::Task['cache:clear'].invoke
 
@@ -133,8 +130,12 @@ namespace :gitlab do
         if ENV["SKIP"] && ENV["SKIP"].include?("db")
           puts_time "[SKIPPED]".color(:cyan)
         else
-          Backup::Database.new(progress).dump
-          puts_time "done".color(:green)
+          begin
+            Backup::Database.new(progress).dump
+            puts_time "done".color(:green)
+          rescue Backup::DatabaseBackupError => e
+            progress.puts "#{e.message}"
+          end
         end
       end
 
@@ -166,8 +167,12 @@ namespace :gitlab do
         if ENV["SKIP"] && ENV["SKIP"].include?("builds")
           puts_time "[SKIPPED]".color(:cyan)
         else
-          Backup::Builds.new(progress).dump
-          puts_time "done".color(:green)
+          begin
+            Backup::Builds.new(progress).dump
+            puts_time "done".color(:green)
+          rescue Backup::FileBackupError => e
+            progress.puts "#{e.message}"
+          end
         end
       end
 
@@ -185,8 +190,12 @@ namespace :gitlab do
         if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
           puts_time "[SKIPPED]".color(:cyan)
         else
-          Backup::Uploads.new(progress).dump
-          puts_time "done".color(:green)
+          begin
+            Backup::Uploads.new(progress).dump
+            puts_time "done".color(:green)
+          rescue Backup::FileBackupError => e
+            progress.puts "#{e.message}"
+          end
         end
       end
 
@@ -204,8 +213,12 @@ namespace :gitlab do
         if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
           puts_time "[SKIPPED]".color(:cyan)
         else
-          Backup::Artifacts.new(progress).dump
-          puts_time "done".color(:green)
+          begin
+            Backup::Artifacts.new(progress).dump
+            puts_time "done".color(:green)
+          rescue Backup::FileBackupError => e
+            progress.puts "#{e.message}"
+          end
         end
       end
 
@@ -223,8 +236,12 @@ namespace :gitlab do
         if ENV["SKIP"] && ENV["SKIP"].include?("pages")
           puts_time "[SKIPPED]".color(:cyan)
         else
-          Backup::Pages.new(progress).dump
-          puts_time "done".color(:green)
+          begin
+            Backup::Pages.new(progress).dump
+            puts_time "done".color(:green)
+          rescue Backup::FileBackupError => e
+            progress.puts "#{e.message}"
+          end
         end
       end
 
@@ -242,8 +259,12 @@ namespace :gitlab do
         if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
           puts_time "[SKIPPED]".color(:cyan)
         else
-          Backup::Lfs.new(progress).dump
-          puts_time "done".color(:green)
+          begin
+            Backup::Lfs.new(progress).dump
+            puts_time "done".color(:green)
+          rescue Backup::FileBackupError => e
+            progress.puts "#{e.message}"
+          end
         end
       end
 
@@ -254,6 +275,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :terraform_state do
+      task create: :gitlab_environment do
+        puts_time "Dumping terraform states ... ".color(:blue)
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state")
+          puts_time "[SKIPPED]".color(:cyan)
+        else
+          Backup::TerraformState.new(progress).dump
+          puts_time "done".color(:green)
+        end
+      end
+
+      task restore: :gitlab_environment do
+        puts_time "Restoring terraform states ... ".color(:blue)
+        Backup::TerraformState.new(progress).restore
+        puts_time "done".color(:green)
+      end
+    end
+
     namespace :registry do
       task create: :gitlab_environment do
         puts_time "Dumping container registry images ... ".color(:blue)
@@ -262,8 +302,12 @@ namespace :gitlab do
           if ENV["SKIP"] && ENV["SKIP"].include?("registry")
             puts_time "[SKIPPED]".color(:cyan)
           else
-            Backup::Registry.new(progress).dump
-            puts_time "done".color(:green)
+            begin
+              Backup::Registry.new(progress).dump
+              puts_time "done".color(:green)
+            rescue Backup::FileBackupError => e
+              progress.puts "#{e.message}"
+            end
           end
         else
           puts_time "[DISABLED]".color(:cyan)
@@ -282,6 +326,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :packages do
+      task create: :gitlab_environment do
+        puts_time "Dumping packages ... ".color(:blue)
+
+        if ENV['SKIP'] && ENV['SKIP'].include?('packages')
+          puts_time "[SKIPPED]".color(:cyan)
+        else
+          Backup::Packages.new(progress).dump
+          puts_time "done".color(:green)
+        end
+      end
+
+      task restore: :gitlab_environment do
+        puts_time "Restoring packages ...".color(:blue)
+        Backup::Packages.new(progress).restore
+        puts_time "done".color(:green)
+      end
+    end
+
     def puts_time(msg)
       progress.puts "#{Time.now} -- #{msg}"
       Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
@@ -302,7 +365,7 @@ namespace :gitlab do
       if Feature.enabled?(:gitaly_backup, default_enabled: :yaml)
         max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
         max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
-        Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency)
+        Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
       else
         Backup::GitalyRpcBackup.new(progress)
       end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 8f033a41e3d..f908a7606fa 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -100,15 +100,13 @@ namespace :gitlab do
     namespace :sessions do
       desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys"
       task active_sessions_lookup_keys: :gitlab_environment do
-        use_redis_session_store = Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true)
-        redis_store_class = use_redis_session_store ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState
         session_key_pattern = "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:*"
         last_save_check = Time.at(0)
         wait_time = 10.seconds
         cursor = 0
         total_users_scanned = 0
 
-        redis_store_class.with do |redis|
+        Gitlab::Redis::Sessions.with do |redis|
           begin
             cursor, keys = redis.scan(cursor, match: session_key_pattern)
             total_users_scanned += keys.count
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 9e733fc3a0f..efb0e1ef1e1 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -170,7 +170,7 @@ namespace :gitlab do
       # the `ActiveRecord::Base.connection` might be switched to another one
       # This is due to `if should_reconnect`:
       # https://github.com/rails/rails/blob/a81aeb63a007ede2fe606c50539417dada9030c7/activerecord/lib/active_record/railties/databases.rake#L622
-      ActiveRecord::Base.establish_connection :main
+      ActiveRecord::Base.establish_connection :main # rubocop: disable Database/EstablishConnection
 
       Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
     end
diff --git a/lib/tasks/gitlab/docs/compile_deprecations.rake b/lib/tasks/gitlab/docs/compile_deprecations.rake
index dc9788cb0b2..4ac68a9f850 100644
--- a/lib/tasks/gitlab/docs/compile_deprecations.rake
+++ b/lib/tasks/gitlab/docs/compile_deprecations.rake
@@ -4,19 +4,19 @@ namespace :gitlab do
   namespace :docs do
     desc "Generate deprecation list from individual files"
     task :compile_deprecations do
-      require_relative '../../../../tooling/deprecations/docs'
-
-      File.write(Deprecations::Docs.path, Deprecations::Docs.render)
-
-      puts "Deprecations compiled to #{Deprecations::Docs.path}"
+      require_relative '../../../../tooling/docs/deprecation_handling'
+      path = Rails.root.join("doc/update/deprecations.md")
+      File.write(path, Docs::DeprecationHandling.new('deprecation').render)
+      puts "Deprecations compiled to #{path}"
     end
 
     desc "Check that the deprecation doc is up to date"
     task :check_deprecations do
-      require_relative '../../../../tooling/deprecations/docs'
+      require_relative '../../../../tooling/docs/deprecation_handling'
+      path = Rails.root.join("doc/update/deprecations.md")
 
-      contents = Deprecations::Docs.render
-      doc = File.read(Deprecations::Docs.path)
+      contents = Docs::DeprecationHandling.new('deprecation').render
+      doc = File.read(path)
 
       if doc == contents
         puts "Deprecations doc is up to date."
@@ -25,5 +25,28 @@ namespace :gitlab do
         abort
       end
     end
+
+    desc "Generate removal list from individual files"
+    task :compile_removals do
+      require_relative '../../../../tooling/docs/deprecation_handling'
+      path = Rails.root.join("doc/update/removals.md")
+      File.write(path, Docs::DeprecationHandling.new('removal').render)
+      puts "Removals compiled to #{path}"
+    end
+
+    desc "Check that the removal doc is up to date"
+    task :check_removals do
+      require_relative '../../../../tooling/docs/deprecation_handling'
+      path = Rails.root.join("doc/update/removals.md")
+      contents = Docs::DeprecationHandling.new('removal').render
+      doc = File.read(path)
+
+      if doc == contents
+        puts "Removals doc is up to date."
+      else
+        format_output('Removals doc is outdated! You (or your technical writer) can update it by running `bin/rake gitlab:docs:compile_removals`.')
+        abort
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake
index 123a4775605..e7ece9e0fdd 100644
--- a/lib/tasks/gitlab/docs/redirect.rake
+++ b/lib/tasks/gitlab/docs/redirect.rake
@@ -51,7 +51,7 @@ namespace :gitlab do
         post.puts "remove_date: '#{date}'"
         post.puts '---'
         post.puts
-        post.puts "This file was moved to [another location](#{new_path})."
+        post.puts "This document was moved to [another location](#{new_path})."
         post.puts
         post.puts "<!-- This redirect file can be deleted after <#{date}>. -->"
         post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->"
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index b01a7902bf2..18c68615637 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -2,41 +2,6 @@
 
 namespace :gitlab do
   namespace :gitaly do
-    desc 'Installs gitaly for running tests within gitlab-development-kit'
-    task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
-      inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT'))
-
-      if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk
-        Rake::Task["gitlab:gitaly:install"].invoke(*args)
-
-        next
-      end
-
-      gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly'))
-
-      # Our test setup expects a git repo, so clone rather than copy
-      clone_repo(gdk_gitaly_dir, args.dir, clone_opts: %w[--depth 1]) unless Dir.exist?(args.dir)
-
-      # We assume the GDK gitaly already compiled binaries
-      build_dir = File.join(gdk_gitaly_dir, '_build')
-      FileUtils.cp_r(build_dir, args.dir)
-
-      # We assume the GDK gitaly already ran bundle install
-      bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle')
-      FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby'))
-
-      # For completeness we copy this for gitaly's make target
-      ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle')
-      FileUtils.cp_r(ruby_bundle_file, args.dir)
-
-      gitaly_binary = File.join(build_dir, 'bin', 'gitaly')
-      warn_gitaly_out_of_date!(gitaly_binary, Gitlab::GitalyClient.expected_server_version)
-    rescue Errno::ENOENT => e
-      puts "Could not copy files, did you run `gdk update`? Error: #{e.message}"
-
-      raise
-    end
-
     desc 'GitLab | Gitaly | Clone and checkout gitaly'
     task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
@@ -60,9 +25,6 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       storage_paths = { 'default' => args.storage_path }
       Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths)
 
-      # In CI we run scripts/gitaly-test-build
-      next if ENV['CI'].present?
-
       Dir.chdir(args.dir) do
         Bundler.with_original_env do
           env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }
diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake
index a9a350fb6c3..491cf782985 100644
--- a/lib/tasks/gitlab/seed/group_seed.rake
+++ b/lib/tasks/gitlab/seed/group_seed.rake
@@ -125,7 +125,7 @@ class GroupSeeder
       name: FFaker::Name.name,
       email: FFaker::Internet.email,
       confirmed_at: DateTime.now,
-      password: Devise.friendly_token
+      password: Gitlab::Password.test_default
     )
   end