Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42

10 files changed, 273 insertions, 73 deletions

diff --git a/lib/tasks/ci/build_artifacts.rake b/lib/tasks/ci/build_artifacts.rake
new file mode 100644
index 00000000000..4f4faef5a62
--- /dev/null
+++ b/lib/tasks/ci/build_artifacts.rake
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'httparty'
+require 'csv'
+
+namespace :ci do
+  namespace :build_artifacts do
+    desc "GitLab | CI | Fetch projects with incorrect artifact size on GitLab.com"
+    task :project_with_incorrect_artifact_size do
+      csv_url = ENV['SISENSE_PROJECT_IDS_WITH_INCORRECT_ARTIFACTS_URL']
+
+      # rubocop: disable Gitlab/HTTParty
+      body = HTTParty.get(csv_url)
+      # rubocop: enable Gitlab/HTTParty
+
+      table = CSV.parse(body.parsed_response, headers: true)
+      puts table['PROJECT_ID'].join(' ')
+    end
+  end
+end
diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake
index cb01f229cd3..99ffeb4ec0b 100644
--- a/lib/tasks/dev.rake
+++ b/lib/tasks/dev.rake
@@ -8,8 +8,10 @@ namespace :dev do
     ENV['force'] = 'yes'
     Rake::Task["gitlab:setup"].invoke
 
-    # Make sure DB statistics are up to date.
-    ActiveRecord::Base.connection.execute('ANALYZE')
+    Gitlab::Database::EachDatabase.each_database_connection do |connection|
+      # Make sure DB statistics are up to date.
+      connection.execute('ANALYZE')
+    end
 
     Rake::Task["gitlab:shell:setup"].invoke
   end
diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake
index c7f3d003f9f..033427fa799 100644
--- a/lib/tasks/gitlab/background_migrations.rake
+++ b/lib/tasks/gitlab/background_migrations.rake
@@ -1,41 +1,110 @@
 # frozen_string_literal: true
 
+databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
+
 namespace :gitlab do
   namespace :background_migrations do
     desc 'Synchronously finish executing a batched background migration'
     task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args|
-      [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument|
-        unless args[argument]
-          puts "Must specify #{argument} as an argument".color(:red)
-          exit 1
-        end
+      if Gitlab::Database.db_config_names.size > 1
+        puts "Please specify the database".color(:red)
+        exit 1
       end
 
-      Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(
+      validate_finalization_arguments!(args)
+
+      main_model = Gitlab::Database.database_base_models[:main]
+
+      finalize_migration(
         args[:job_class_name],
         args[:table_name],
         args[:column_name],
-        Gitlab::Json.parse(args[:job_arguments])
+        Gitlab::Json.parse(args[:job_arguments]),
+        connection: main_model.connection
       )
+    end
 
-      puts "Done.".color(:green)
+    namespace :finalize do
+      ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
+        next if name.to_s == 'geo'
+
+        desc "Gitlab | DB | Synchronously finish executing a batched background migration on #{name} database"
+        task name, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args|
+          validate_finalization_arguments!(args)
+
+          model = Gitlab::Database.database_base_models[name]
+
+          finalize_migration(
+            args[:job_class_name],
+            args[:table_name],
+            args[:column_name],
+            Gitlab::Json.parse(args[:job_arguments]),
+            connection: model.connection
+          )
+        end
+      end
     end
 
     desc 'Display the status of batched background migrations'
-    task status: :environment do
-      statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses
-      max_status_length = statuses.keys.map(&:length).max
-      format_string = "%-#{max_status_length}s | %s\n"
-
-      Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration|
-        identification_fields = [
-          migration.job_class_name,
-          migration.table_name,
-          migration.column_name,
-          migration.job_arguments.to_json
-        ].join(',')
-
-        printf(format_string, migration.status, identification_fields)
+    task status: :environment do |_, args|
+      Gitlab::Database.database_base_models.each do |name, model|
+        display_migration_status(name, model.connection)
+      end
+    end
+
+    namespace :status do
+      ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
+        next if name.to_s == 'geo'
+
+        desc "Gitlab | DB | Display the status of batched background migrations on #{name} database"
+        task name => :environment do |_, args|
+          model = Gitlab::Database.database_base_models[name]
+          display_migration_status(name, model.connection)
+        end
+      end
+    end
+
+    private
+
+    def finalize_migration(class_name, table_name, column_name, job_arguments, connection:)
+      Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(
+        class_name,
+        table_name,
+        column_name,
+        Gitlab::Json.parse(job_arguments),
+        connection: connection
+      )
+
+      puts "Done.".color(:green)
+    end
+
+    def display_migration_status(database_name, connection)
+      Gitlab::Database::SharedModel.using_connection(connection) do
+        statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses
+        max_status_length = statuses.keys.map(&:length).max
+        format_string = "%-#{max_status_length}s | %s\n"
+
+        puts "Database: #{database_name}\n"
+
+        Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration|
+          identification_fields = [
+            migration.job_class_name,
+            migration.table_name,
+            migration.column_name,
+            migration.job_arguments.to_json
+          ].join(',')
+
+          printf(format_string, migration.status, identification_fields)
+        end
+      end
+    end
+
+    def validate_finalization_arguments!(args)
+      [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument|
+        unless args[argument]
+          puts "Must specify #{argument} as an argument".color(:red)
+          exit 1
+        end
       end
     end
   end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 6d4af9d166f..50ceb11581e 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -4,30 +4,28 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
 
 namespace :gitlab do
   namespace :db do
-    desc 'GitLab | DB | Manually insert schema migration version'
+    desc 'GitLab | DB | Manually insert schema migration version on all configured databases'
     task :mark_migration_complete, [:version] => :environment do |_, args|
       mark_migration_complete(args[:version])
     end
 
     namespace :mark_migration_complete do
-      ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
-        desc "Gitlab | DB | Manually insert schema migration version on #{name} database"
-        task name, [:version] => :environment do |_, args|
-          mark_migration_complete(args[:version], database: name)
+      ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database|
+        desc "Gitlab | DB | Manually insert schema migration version on #{database} database"
+        task database, [:version] => :environment do |_, args|
+          mark_migration_complete(args[:version], only_on: database)
         end
       end
     end
 
-    def mark_migration_complete(version, database: nil)
+    def mark_migration_complete(version, only_on: nil)
       if version.to_i == 0
         puts 'Must give a version argument that is a non-zero integer'.color(:red)
         exit 1
       end
 
-      Gitlab::Database.database_base_models.each do |name, model|
-        next if database && database.to_s != name
-
-        model.connection.execute("INSERT INTO schema_migrations (version) VALUES (#{model.connection.quote(version)})")
+      Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name|
+        connection.execute("INSERT INTO schema_migrations (version) VALUES (#{connection.quote(version)})")
 
         puts "Successfully marked '#{version}' as complete on database #{name}".color(:green)
       rescue ActiveRecord::RecordNotUnique
@@ -35,32 +33,44 @@ namespace :gitlab do
       end
     end
 
-    desc 'GitLab | DB | Drop all tables'
+    desc 'GitLab | DB | Drop all tables on all configured databases'
     task drop_tables: :environment do
-      connection = ActiveRecord::Base.connection
+      drop_tables
+    end
 
-      # In PostgreSQLAdapter, data_sources returns both views and tables, so use
-      # #tables instead
-      tables = connection.tables
+    namespace :drop_tables do
+      ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database|
+        desc "GitLab | DB | Drop all tables on the #{database} database"
+        task database => :environment do
+          drop_tables(only_on: database)
+        end
+      end
+    end
 
-      # Removes the entry from the array
-      tables.delete 'schema_migrations'
-      # Truncate schema_migrations to ensure migrations re-run
-      connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'
+    def drop_tables(only_on: nil)
+      Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name|
+        # In PostgreSQLAdapter, data_sources returns both views and tables, so use tables instead
+        tables = connection.tables
 
-      # Drop any views
-      connection.views.each do |view|
-        connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE")
-      end
+        # Removes the entry from the array
+        tables.delete 'schema_migrations'
+        # Truncate schema_migrations to ensure migrations re-run
+        connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'
 
-      # Drop tables with cascade to avoid dependent table errors
-      # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
-      # Add `IF EXISTS` because cascade could have already deleted a table.
-      tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
+        # Drop any views
+        connection.views.each do |view|
+          connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE")
+        end
 
-      # Drop all extra schema objects GitLab owns
-      Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
-        connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE")
+        # Drop tables with cascade to avoid dependent table errors
+        # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
+        # Add `IF EXISTS` because cascade could have already deleted a table.
+        tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
+
+        # Drop all extra schema objects GitLab owns
+        Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
+          connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE")
+        end
       end
     end
 
@@ -152,6 +162,17 @@ namespace :gitlab do
       Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
     end
 
+    ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
+      # We'll temporarily skip this enhancement for geo, since in some situations we
+      # wish to setup the geo database before the other databases have been setup,
+      # and partition management attempts to connect to the main database.
+      next if name == 'geo'
+
+      Rake::Task["db:migrate:#{name}"].enhance do
+        Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
+      end
+    end
+
     # When we load the database schema from db/structure.sql
     # we don't have any dynamic partitions created. We don't really need to
     # because application initializers/sidekiq take care of that, too.
@@ -160,16 +181,29 @@ namespace :gitlab do
     #
     # Other than that it's helpful to create partitions early when bootstrapping
     # a new installation.
-    #
-    # Rails 6.1 deprecates db:structure:load in favor of db:schema:load
-    Rake::Task['db:structure:load'].enhance do
+    Rake::Task['db:schema:load'].enhance do
       Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
     end
 
-    Rake::Task['db:schema:load'].enhance do
-      Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
+    ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
+      # We'll temporarily skip this enhancement for geo, since in some situations we
+      # wish to setup the geo database before the other databases have been setup,
+      # and partition management attempts to connect to the main database.
+      next if name == 'geo'
+
+      Rake::Task["db:schema:load:#{name}"].enhance do
+        Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
+      end
+    end
+
+    desc "Clear all connections"
+    task :clear_all_connections do
+      ActiveRecord::Base.clear_all_connections!
     end
 
+    Rake::Task['db:test:purge'].enhance(['gitlab:db:clear_all_connections'])
+    Rake::Task['db:drop'].enhance(['gitlab:db:clear_all_connections'])
+
     # During testing, db:test:load restores the database schema from scratch
     # which does not include dynamic partitions. We cannot rely on application
     # initializers here as the application can continue to run while
@@ -195,8 +229,6 @@ namespace :gitlab do
     end
 
     namespace :reindex do
-      databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
-
       ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name|
         desc "Reindex #{database_name} database without downtime to eliminate bloat"
         task database_name => :environment do
diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake
index e7ece9e0fdd..2d234fcdb36 100644
--- a/lib/tasks/gitlab/docs/redirect.rake
+++ b/lib/tasks/gitlab/docs/redirect.rake
@@ -54,7 +54,9 @@ namespace :gitlab do
         post.puts "This document was moved to [another location](#{new_path})."
         post.puts
         post.puts "<!-- This redirect file can be deleted after <#{date}>. -->"
-        post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->"
+        post.puts "<!-- Redirects that point to other docs in the same project expire in three months. -->"
+        post.puts "<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. -->"
+        post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->"
       end
     end
   end
diff --git a/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake
new file mode 100644
index 00000000000..1cc18d14d78
--- /dev/null
+++ b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  desc "GitLab | Refresh build artifacts size project statistics for given project IDs"
+
+  BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500
+
+  task :refresh_project_statistics_build_artifacts_size, [:project_ids] => :environment do |_t, args|
+    project_ids = []
+    project_ids = $stdin.read.split unless $stdin.tty?
+    project_ids = args.project_ids.to_s.split unless project_ids.any?
+
+    if project_ids.any?
+      project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE) do |ids|
+        projects = Project.where(id: ids)
+        Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects)
+      end
+      puts 'Done.'.green
+    else
+      puts 'Please provide a string of space-separated project IDs as the argument or through the STDIN'.red
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index 705519d1741..a5289476378 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -47,13 +47,15 @@ namespace :gitlab do
   # will work.
   def self.terminate_all_connections
     cmd = <<~SQL
-    SELECT pg_terminate_backend(pg_stat_activity.pid)
-        FROM pg_stat_activity
-        WHERE datname = current_database()
+      SELECT pg_terminate_backend(pg_stat_activity.pid)
+      FROM pg_stat_activity
+      WHERE datname = current_database()
         AND pid <> pg_backend_pid();
     SQL
 
-    ActiveRecord::Base.connection.execute(cmd)&.result_status == PG::PGRES_TUPLES_OK
-  rescue ActiveRecord::NoDatabaseError
+    Gitlab::Database::EachDatabase.each_database_connection do |connection|
+      connection.execute(cmd)
+    rescue ActiveRecord::NoDatabaseError
+    end
   end
 end
diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake
index 43fd4f8685a..358bc6c31eb 100644
--- a/lib/tasks/gitlab/tw/codeowners.rake
+++ b/lib/tasks/gitlab/tw/codeowners.rake
@@ -22,7 +22,7 @@ namespace :tw do
       CodeOwnerRule.new('Container Security', '@ngaskill'),
       CodeOwnerRule.new('Contributor Experience', '@eread'),
       CodeOwnerRule.new('Conversion', '@kpaizee'),
-      CodeOwnerRule.new('Database', '@marcia'),
+      CodeOwnerRule.new('Database', '@aqualls'),
       CodeOwnerRule.new('Development', '@marcia'),
       CodeOwnerRule.new('Distribution', '@axil'),
       CodeOwnerRule.new('Distribution (Charts)', '@axil'),
diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake
index 8c5edb5de8a..6eabdf51dcd 100644
--- a/lib/tasks/rubocop.rake
+++ b/lib/tasks/rubocop.rake
@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# rubocop:disable Rails/RakeEnvironment
 
 unless Rails.env.production?
   require 'rubocop/rake_task'
@@ -8,18 +9,59 @@ unless Rails.env.production?
   namespace :rubocop do
     namespace :todo do
       desc 'Generate RuboCop todos'
-      task :generate do # rubocop:disable Rails/RakeEnvironment
+      task :generate do |_task, args|
         require 'rubocop'
+        require 'active_support/inflector/inflections'
+        require_relative '../../rubocop/todo_dir'
+        require_relative '../../rubocop/formatter/todo_formatter'
+
+        # Reveal all pending TODOs so RuboCop can pick them up and report
+        # during scan.
+        ENV['REVEAL_RUBOCOP_TODO'] = '1'
+
+        # Save cop configuration like `RSpec/ContextWording` into
+        # `rspec/context_wording.yml` and not into
+        # `r_spec/context_wording.yml`.
+        ActiveSupport::Inflector.inflections(:en) do |inflect|
+          inflect.acronym 'RSpec'
+          inflect.acronym 'GraphQL'
+        end
 
         options = %w[
-          --auto-gen-config
-          --auto-gen-only-exclude
-          --exclude-limit=100000
-          --no-offense-counts
+          --parallel
+          --format RuboCop::Formatter::TodoFormatter
         ]
 
+        # Convert from Rake::TaskArguments into an Array to make `any?` work as
+        # expected.
+        cop_names = args.to_a
+
+        todo_dir = RuboCop::TodoDir.new(RuboCop::TodoDir::DEFAULT_TODO_DIR)
+
+        if cop_names.any?
+          # We are sorting the cop names to benefit from RuboCop cache which
+          # also takes passed parameters into account.
+          list = cop_names.sort.join(',')
+          options.concat ['--only', list]
+
+          cop_names.each { |cop_name| todo_dir.inspect(cop_name) }
+        else
+          todo_dir.inspect_all
+        end
+
+        puts <<~MSG
+          Generating RuboCop TODOs with:
+            rubocop #{options.join(' ')}
+
+          This might take a while...
+        MSG
+
         RuboCop::CLI.new.run(options)
+
+        todo_dir.delete_inspected
       end
     end
   end
 end
+
+# rubocop:enable Rails/RakeEnvironment
diff --git a/lib/tasks/tanuki_emoji.rake b/lib/tasks/tanuki_emoji.rake
index 98d3920c07f..0dc7dd4e701 100644
--- a/lib/tasks/tanuki_emoji.rake
+++ b/lib/tasks/tanuki_emoji.rake
@@ -3,12 +3,20 @@
 namespace :tanuki_emoji do
   desc 'Generates Emoji aliases fixtures'
   task aliases: :environment do
+    ALLOWED_ALIASES = [':)', ':('].freeze
     aliases = {}
 
     TanukiEmoji.index.all.each do |emoji|
       emoji.aliases.each do |emoji_alias|
         aliases[TanukiEmoji::Character.format_name(emoji_alias)] = emoji.name
       end
+
+      emoji.ascii_aliases.intersection(ALLOWED_ALIASES).each do |ascii_alias|
+        # We add an extra space at the end so that when a user types ":) "
+        # we'd still match this alias and not show "cocos (keeling) islands" as the first result.
+        # The initial ":" is ignored when matching because it's our emoji prefix in Markdown.
+        aliases[ascii_alias + ' '] = emoji.name
+      end
     end
 
     aliases_json_file = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')