From 41fe97390ceddf945f3d967b8fdb3de4c66b7dea Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 18 Mar 2022 20:02:30 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-9-stable-ee --- lib/tasks/ci/build_artifacts.rake | 20 ++++ lib/tasks/dev.rake | 6 +- lib/tasks/gitlab/background_migrations.rake | 113 +++++++++++++++++---- lib/tasks/gitlab/db.rake | 106 ++++++++++++------- lib/tasks/gitlab/docs/redirect.rake | 4 +- ...sh_project_statistics_build_artifacts_size.rake | 23 +++++ lib/tasks/gitlab/setup.rake | 12 ++- lib/tasks/gitlab/tw/codeowners.rake | 2 +- lib/tasks/rubocop.rake | 52 +++++++++- lib/tasks/tanuki_emoji.rake | 8 ++ 10 files changed, 273 insertions(+), 73 deletions(-) create mode 100644 lib/tasks/ci/build_artifacts.rake create mode 100644 lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake (limited to 'lib/tasks') diff --git a/lib/tasks/ci/build_artifacts.rake b/lib/tasks/ci/build_artifacts.rake new file mode 100644 index 00000000000..4f4faef5a62 --- /dev/null +++ b/lib/tasks/ci/build_artifacts.rake @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'httparty' +require 'csv' + +namespace :ci do + namespace :build_artifacts do + desc "GitLab | CI | Fetch projects with incorrect artifact size on GitLab.com" + task :project_with_incorrect_artifact_size do + csv_url = ENV['SISENSE_PROJECT_IDS_WITH_INCORRECT_ARTIFACTS_URL'] + + # rubocop: disable Gitlab/HTTParty + body = HTTParty.get(csv_url) + # rubocop: enable Gitlab/HTTParty + + table = CSV.parse(body.parsed_response, headers: true) + puts table['PROJECT_ID'].join(' ') + end + end +end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index cb01f229cd3..99ffeb4ec0b 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -8,8 +8,10 @@ namespace :dev do ENV['force'] = 'yes' Rake::Task["gitlab:setup"].invoke - # Make sure DB statistics are up to date. - ActiveRecord::Base.connection.execute('ANALYZE') + Gitlab::Database::EachDatabase.each_database_connection do |connection| + # Make sure DB statistics are up to date. + connection.execute('ANALYZE') + end Rake::Task["gitlab:shell:setup"].invoke end diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake index c7f3d003f9f..033427fa799 100644 --- a/lib/tasks/gitlab/background_migrations.rake +++ b/lib/tasks/gitlab/background_migrations.rake @@ -1,41 +1,110 @@ # frozen_string_literal: true +databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml + namespace :gitlab do namespace :background_migrations do desc 'Synchronously finish executing a batched background migration' task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| - [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| - unless args[argument] - puts "Must specify #{argument} as an argument".color(:red) - exit 1 - end + if Gitlab::Database.db_config_names.size > 1 + puts "Please specify the database".color(:red) + exit 1 end - Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize( + validate_finalization_arguments!(args) + + main_model = Gitlab::Database.database_base_models[:main] + + finalize_migration( args[:job_class_name], args[:table_name], args[:column_name], - Gitlab::Json.parse(args[:job_arguments]) + Gitlab::Json.parse(args[:job_arguments]), + connection: main_model.connection ) + end - puts "Done.".color(:green) + namespace :finalize do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + next if name.to_s == 'geo' + + desc "Gitlab | DB | Synchronously finish executing a batched background migration on #{name} database" + task name, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| + validate_finalization_arguments!(args) + + model = Gitlab::Database.database_base_models[name] + + finalize_migration( + args[:job_class_name], + args[:table_name], + args[:column_name], + Gitlab::Json.parse(args[:job_arguments]), + connection: model.connection + ) + end + end end desc 'Display the status of batched background migrations' - task status: :environment do - statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses - max_status_length = statuses.keys.map(&:length).max - format_string = "%-#{max_status_length}s | %s\n" - - Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration| - identification_fields = [ - migration.job_class_name, - migration.table_name, - migration.column_name, - migration.job_arguments.to_json - ].join(',') - - printf(format_string, migration.status, identification_fields) + task status: :environment do |_, args| + Gitlab::Database.database_base_models.each do |name, model| + display_migration_status(name, model.connection) + end + end + + namespace :status do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + next if name.to_s == 'geo' + + desc "Gitlab | DB | Display the status of batched background migrations on #{name} database" + task name => :environment do |_, args| + model = Gitlab::Database.database_base_models[name] + display_migration_status(name, model.connection) + end + end + end + + private + + def finalize_migration(class_name, table_name, column_name, job_arguments, connection:) + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize( + class_name, + table_name, + column_name, + Gitlab::Json.parse(job_arguments), + connection: connection + ) + + puts "Done.".color(:green) + end + + def display_migration_status(database_name, connection) + Gitlab::Database::SharedModel.using_connection(connection) do + statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses + max_status_length = statuses.keys.map(&:length).max + format_string = "%-#{max_status_length}s | %s\n" + + puts "Database: #{database_name}\n" + + Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration| + identification_fields = [ + migration.job_class_name, + migration.table_name, + migration.column_name, + migration.job_arguments.to_json + ].join(',') + + printf(format_string, migration.status, identification_fields) + end + end + end + + def validate_finalization_arguments!(args) + [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| + unless args[argument] + puts "Must specify #{argument} as an argument".color(:red) + exit 1 + end end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 6d4af9d166f..50ceb11581e 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -4,30 +4,28 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml namespace :gitlab do namespace :db do - desc 'GitLab | DB | Manually insert schema migration version' + desc 'GitLab | DB | Manually insert schema migration version on all configured databases' task :mark_migration_complete, [:version] => :environment do |_, args| mark_migration_complete(args[:version]) end namespace :mark_migration_complete do - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| - desc "Gitlab | DB | Manually insert schema migration version on #{name} database" - task name, [:version] => :environment do |_, args| - mark_migration_complete(args[:version], database: name) + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| + desc "Gitlab | DB | Manually insert schema migration version on #{database} database" + task database, [:version] => :environment do |_, args| + mark_migration_complete(args[:version], only_on: database) end end end - def mark_migration_complete(version, database: nil) + def mark_migration_complete(version, only_on: nil) if version.to_i == 0 puts 'Must give a version argument that is a non-zero integer'.color(:red) exit 1 end - Gitlab::Database.database_base_models.each do |name, model| - next if database && database.to_s != name - - model.connection.execute("INSERT INTO schema_migrations (version) VALUES (#{model.connection.quote(version)})") + Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name| + connection.execute("INSERT INTO schema_migrations (version) VALUES (#{connection.quote(version)})") puts "Successfully marked '#{version}' as complete on database #{name}".color(:green) rescue ActiveRecord::RecordNotUnique @@ -35,32 +33,44 @@ namespace :gitlab do end end - desc 'GitLab | DB | Drop all tables' + desc 'GitLab | DB | Drop all tables on all configured databases' task drop_tables: :environment do - connection = ActiveRecord::Base.connection + drop_tables + end - # In PostgreSQLAdapter, data_sources returns both views and tables, so use - # #tables instead - tables = connection.tables + namespace :drop_tables do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| + desc "GitLab | DB | Drop all tables on the #{database} database" + task database => :environment do + drop_tables(only_on: database) + end + end + end - # Removes the entry from the array - tables.delete 'schema_migrations' - # Truncate schema_migrations to ensure migrations re-run - connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' + def drop_tables(only_on: nil) + Gitlab::Database::EachDatabase.each_database_connection(only: only_on) do |connection, name| + # In PostgreSQLAdapter, data_sources returns both views and tables, so use tables instead + tables = connection.tables - # Drop any views - connection.views.each do |view| - connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE") - end + # Removes the entry from the array + tables.delete 'schema_migrations' + # Truncate schema_migrations to ensure migrations re-run + connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' - # Drop tables with cascade to avoid dependent table errors - # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html - # Add `IF EXISTS` because cascade could have already deleted a table. - tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } + # Drop any views + connection.views.each do |view| + connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE") + end - # Drop all extra schema objects GitLab owns - Gitlab::Database::EXTRA_SCHEMAS.each do |schema| - connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE") + # Drop tables with cascade to avoid dependent table errors + # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html + # Add `IF EXISTS` because cascade could have already deleted a table. + tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } + + # Drop all extra schema objects GitLab owns + Gitlab::Database::EXTRA_SCHEMAS.each do |schema| + connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE") + end end end @@ -152,6 +162,17 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + # We'll temporarily skip this enhancement for geo, since in some situations we + # wish to setup the geo database before the other databases have been setup, + # and partition management attempts to connect to the main database. + next if name == 'geo' + + Rake::Task["db:migrate:#{name}"].enhance do + Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + end + end + # When we load the database schema from db/structure.sql # we don't have any dynamic partitions created. We don't really need to # because application initializers/sidekiq take care of that, too. @@ -160,16 +181,29 @@ namespace :gitlab do # # Other than that it's helpful to create partitions early when bootstrapping # a new installation. - # - # Rails 6.1 deprecates db:structure:load in favor of db:schema:load - Rake::Task['db:structure:load'].enhance do + Rake::Task['db:schema:load'].enhance do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - Rake::Task['db:schema:load'].enhance do - Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + # We'll temporarily skip this enhancement for geo, since in some situations we + # wish to setup the geo database before the other databases have been setup, + # and partition management attempts to connect to the main database. + next if name == 'geo' + + Rake::Task["db:schema:load:#{name}"].enhance do + Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + end + end + + desc "Clear all connections" + task :clear_all_connections do + ActiveRecord::Base.clear_all_connections! end + Rake::Task['db:test:purge'].enhance(['gitlab:db:clear_all_connections']) + Rake::Task['db:drop'].enhance(['gitlab:db:clear_all_connections']) + # During testing, db:test:load restores the database schema from scratch # which does not include dynamic partitions. We cannot rely on application # initializers here as the application can continue to run while @@ -195,8 +229,6 @@ namespace :gitlab do end namespace :reindex do - databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name| desc "Reindex #{database_name} database without downtime to eliminate bloat" task database_name => :environment do diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index e7ece9e0fdd..2d234fcdb36 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -54,7 +54,9 @@ namespace :gitlab do post.puts "This document was moved to [another location](#{new_path})." post.puts post.puts "" - post.puts "" + post.puts "" + post.puts "" + post.puts "" end end end diff --git a/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake new file mode 100644 index 00000000000..1cc18d14d78 --- /dev/null +++ b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + desc "GitLab | Refresh build artifacts size project statistics for given project IDs" + + BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500 + + task :refresh_project_statistics_build_artifacts_size, [:project_ids] => :environment do |_t, args| + project_ids = [] + project_ids = $stdin.read.split unless $stdin.tty? + project_ids = args.project_ids.to_s.split unless project_ids.any? + + if project_ids.any? + project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE) do |ids| + projects = Project.where(id: ids) + Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects) + end + puts 'Done.'.green + else + puts 'Please provide a string of space-separated project IDs as the argument or through the STDIN'.red + end + end +end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 705519d1741..a5289476378 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -47,13 +47,15 @@ namespace :gitlab do # will work. def self.terminate_all_connections cmd = <<~SQL - SELECT pg_terminate_backend(pg_stat_activity.pid) - FROM pg_stat_activity - WHERE datname = current_database() + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid(); SQL - ActiveRecord::Base.connection.execute(cmd)&.result_status == PG::PGRES_TUPLES_OK - rescue ActiveRecord::NoDatabaseError + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute(cmd) + rescue ActiveRecord::NoDatabaseError + end end end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 43fd4f8685a..358bc6c31eb 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -22,7 +22,7 @@ namespace :tw do CodeOwnerRule.new('Container Security', '@ngaskill'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), - CodeOwnerRule.new('Database', '@marcia'), + CodeOwnerRule.new('Database', '@aqualls'), CodeOwnerRule.new('Development', '@marcia'), CodeOwnerRule.new('Distribution', '@axil'), CodeOwnerRule.new('Distribution (Charts)', '@axil'), diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index 8c5edb5de8a..6eabdf51dcd 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -1,4 +1,5 @@ # frozen_string_literal: true +# rubocop:disable Rails/RakeEnvironment unless Rails.env.production? require 'rubocop/rake_task' @@ -8,18 +9,59 @@ unless Rails.env.production? namespace :rubocop do namespace :todo do desc 'Generate RuboCop todos' - task :generate do # rubocop:disable Rails/RakeEnvironment + task :generate do |_task, args| require 'rubocop' + require 'active_support/inflector/inflections' + require_relative '../../rubocop/todo_dir' + require_relative '../../rubocop/formatter/todo_formatter' + + # Reveal all pending TODOs so RuboCop can pick them up and report + # during scan. + ENV['REVEAL_RUBOCOP_TODO'] = '1' + + # Save cop configuration like `RSpec/ContextWording` into + # `rspec/context_wording.yml` and not into + # `r_spec/context_wording.yml`. + ActiveSupport::Inflector.inflections(:en) do |inflect| + inflect.acronym 'RSpec' + inflect.acronym 'GraphQL' + end options = %w[ - --auto-gen-config - --auto-gen-only-exclude - --exclude-limit=100000 - --no-offense-counts + --parallel + --format RuboCop::Formatter::TodoFormatter ] + # Convert from Rake::TaskArguments into an Array to make `any?` work as + # expected. + cop_names = args.to_a + + todo_dir = RuboCop::TodoDir.new(RuboCop::TodoDir::DEFAULT_TODO_DIR) + + if cop_names.any? + # We are sorting the cop names to benefit from RuboCop cache which + # also takes passed parameters into account. + list = cop_names.sort.join(',') + options.concat ['--only', list] + + cop_names.each { |cop_name| todo_dir.inspect(cop_name) } + else + todo_dir.inspect_all + end + + puts <<~MSG + Generating RuboCop TODOs with: + rubocop #{options.join(' ')} + + This might take a while... + MSG + RuboCop::CLI.new.run(options) + + todo_dir.delete_inspected end end end end + +# rubocop:enable Rails/RakeEnvironment diff --git a/lib/tasks/tanuki_emoji.rake b/lib/tasks/tanuki_emoji.rake index 98d3920c07f..0dc7dd4e701 100644 --- a/lib/tasks/tanuki_emoji.rake +++ b/lib/tasks/tanuki_emoji.rake @@ -3,12 +3,20 @@ namespace :tanuki_emoji do desc 'Generates Emoji aliases fixtures' task aliases: :environment do + ALLOWED_ALIASES = [':)', ':('].freeze aliases = {} TanukiEmoji.index.all.each do |emoji| emoji.aliases.each do |emoji_alias| aliases[TanukiEmoji::Character.format_name(emoji_alias)] = emoji.name end + + emoji.ascii_aliases.intersection(ALLOWED_ALIASES).each do |ascii_alias| + # We add an extra space at the end so that when a user types ":) " + # we'd still match this alias and not show "cocos (keeling) islands" as the first result. + # The initial ":" is ignored when matching because it's our emoji prefix in Markdown. + aliases[ascii_alias + ' '] = emoji.name + end end aliases_json_file = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') -- cgit v1.2.1