summaryrefslogtreecommitdiff
path: root/lib/tasks
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-07-20 09:55:51 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2021-07-20 09:55:51 +0000
commite8d2c2579383897a1dd7f9debd359abe8ae8373d (patch)
treec42be41678c2586d49a75cabce89322082698334 /lib/tasks
parentfc845b37ec3a90aaa719975f607740c22ba6a113 (diff)
downloadgitlab-ce-e8d2c2579383897a1dd7f9debd359abe8ae8373d.tar.gz
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to 'lib/tasks')
-rw-r--r--lib/tasks/gitlab/assets.rake4
-rw-r--r--lib/tasks/gitlab/background_migrations.rake23
-rw-r--r--lib/tasks/gitlab/backup.rake5
-rw-r--r--lib/tasks/gitlab/db.rake58
-rw-r--r--lib/tasks/gitlab/helpers.rake4
-rw-r--r--lib/tasks/gitlab/sidekiq.rake25
-rw-r--r--lib/tasks/gitlab/usage_data.rake2
7 files changed, 53 insertions, 68 deletions
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index 54e74fd9c8b..db10428e0dc 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -5,8 +5,8 @@ require 'fileutils'
module Tasks
module Gitlab
module Assets
- FOSS_ASSET_FOLDERS = %w[app/assets app/views fixtures/emojis vendor/assets/javascripts].freeze
- EE_ASSET_FOLDERS = %w[ee/app/assets ee/app/views].freeze
+ FOSS_ASSET_FOLDERS = %w[app/assets fixtures/emojis vendor/assets/javascripts].freeze
+ EE_ASSET_FOLDERS = %w[ee/app/assets].freeze
JS_ASSET_PATTERNS = %w[*.js config/**/*.js].freeze
JS_ASSET_FILES = %w[package.json yarn.lock].freeze
MASTER_MD5_HASH_FILE = 'master-assets-hash.txt'
diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake
new file mode 100644
index 00000000000..c978a2807ca
--- /dev/null
+++ b/lib/tasks/gitlab/background_migrations.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+ namespace :background_migrations do
+ task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args|
+ [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument|
+ unless args[argument]
+ puts "Must specify #{argument} as an argument".color(:red)
+ exit 1
+ end
+ end
+
+ Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(
+ args[:job_class_name],
+ args[:table_name],
+ args[:column_name],
+ Gitlab::Json.parse(args[:job_arguments])
+ )
+
+ puts "Done.".color(:green)
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 5b17a8c185a..ed74dd472ff 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -282,6 +282,7 @@ namespace :gitlab do
def puts_time(msg)
progress.puts "#{Time.now} -- #{msg}"
+ Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
end
def progress
@@ -297,7 +298,9 @@ namespace :gitlab do
def repository_backup_strategy
if Feature.enabled?(:gitaly_backup)
- Backup::GitalyBackup.new(progress)
+ max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
+ max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
+ Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency)
else
Backup::GitalyRpcBackup.new(progress)
end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index ee986f4c503..2b508b341dd 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -90,73 +90,35 @@ namespace :gitlab do
desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump'
task :clean_structure_sql do |task_name|
- structure_file = 'db/structure.sql'
- schema = File.read(structure_file)
+ ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config|
+ structure_file = ActiveRecord::Tasks::DatabaseTasks.dump_filename(db_config.name)
- File.open(structure_file, 'wb+') do |io|
- Gitlab::Database::SchemaCleaner.new(schema).clean(io)
- end
-
- # Allow this task to be called multiple times, as happens when running db:migrate:redo
- Rake::Task[task_name].reenable
- end
+ schema = File.read(structure_file)
- desc 'This dumps GitLab specific database details - it runs after db:structure:dump'
- task :dump_custom_structure do |task_name|
- Gitlab::Database::CustomStructure.new.dump
+ File.open(structure_file, 'wb+') do |io|
+ Gitlab::Database::SchemaCleaner.new(schema).clean(io)
+ end
+ end
# Allow this task to be called multiple times, as happens when running db:migrate:redo
Rake::Task[task_name].reenable
end
- desc 'This loads GitLab specific database details - runs after db:structure:dump'
- task :load_custom_structure do
- configuration = Rails.application.config_for(:database)
-
- ENV['PGHOST'] = configuration['host'] if configuration['host']
- ENV['PGPORT'] = configuration['port'].to_s if configuration['port']
- ENV['PGPASSWORD'] = configuration['password'].to_s if configuration['password']
- ENV['PGUSER'] = configuration['username'].to_s if configuration['username']
-
- command = 'psql'
- dump_filepath = Gitlab::Database::CustomStructure.custom_dump_filepath.to_path
- args = ['-v', 'ON_ERROR_STOP=1', '-q', '-X', '-f', dump_filepath, configuration['database']]
-
- unless Kernel.system(command, *args)
- raise "failed to execute:\n#{command} #{args.join(' ')}\n\n" \
- "Please ensure `#{command}` is installed in your PATH and has proper permissions.\n\n"
- end
- end
-
# Inform Rake that custom tasks should be run every time rake db:structure:dump is run
#
# Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump
Rake::Task['db:structure:dump'].enhance do
Rake::Task['gitlab:db:clean_structure_sql'].invoke
- Rake::Task['gitlab:db:dump_custom_structure'].invoke
end
# Inform Rake that custom tasks should be run every time rake db:schema:dump is run
Rake::Task['db:schema:dump'].enhance do
Rake::Task['gitlab:db:clean_structure_sql'].invoke
- Rake::Task['gitlab:db:dump_custom_structure'].invoke
- end
-
- # Inform Rake that custom tasks should be run every time rake db:structure:load is run
- #
- # Rails 6.1 deprecates db:structure:load in favor of db:schema:load
- Rake::Task['db:structure:load'].enhance do
- Rake::Task['gitlab:db:load_custom_structure'].invoke
- end
-
- # Inform Rake that custom tasks should be run every time rake db:schema:load is run
- Rake::Task['db:schema:load'].enhance do
- Rake::Task['gitlab:db:load_custom_structure'].invoke
end
desc 'Create missing dynamic database partitions'
- task :create_dynamic_partitions do
- Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions
+ task create_dynamic_partitions: :environment do
+ Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions
end
# This is targeted towards deploys and upgrades of GitLab.
@@ -192,7 +154,7 @@ namespace :gitlab do
Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
end
- desc 'reindex a regular (non-unique) index without downtime to eliminate bloat'
+ desc 'reindex a regular index without downtime to eliminate bloat'
task :reindex, [:index_name] => :environment do |_, args|
unless Feature.enabled?(:database_reindexing, type: :ops)
puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake
index b61b1833c5a..b467aa3819d 100644
--- a/lib/tasks/gitlab/helpers.rake
+++ b/lib/tasks/gitlab/helpers.rake
@@ -3,6 +3,8 @@
# Prevent StateMachine warnings from outputting during a cron task
StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
-task gitlab_environment: :environment do
+task :gitlab_environment do
+ Rake::Task[:environment].invoke unless ENV['SKIP_RAILS_ENV_IN_RAKE']
+
extend SystemCheck::Helpers
end
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 6f5c3a86dd3..d3060d92e88 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -1,7 +1,5 @@
# frozen_string_literal: true
-return if Rails.env.production?
-
namespace :gitlab do
namespace :sidekiq do
def write_yaml(path, banner, object)
@@ -9,31 +7,28 @@ namespace :gitlab do
end
namespace :migrate_jobs do
- def mappings
- ::Gitlab::SidekiqConfig
- .workers
- .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) }
- .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] }
- end
-
desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names'
task schedule: :environment do
::Gitlab::SidekiqMigrateJobs
.new('schedule', logger: Logger.new($stdout))
- .execute(mappings)
+ .execute(::Gitlab::SidekiqConfig.worker_queue_mappings)
end
desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names'
task retry: :environment do
::Gitlab::SidekiqMigrateJobs
.new('retry', logger: Logger.new($stdout))
- .execute(mappings)
+ .execute(::Gitlab::SidekiqConfig.worker_queue_mappings)
end
end
+ task :not_production do
+ raise 'This task cannot be run in the production environment' if Rails.env.production?
+ end
+
namespace :all_queues_yml do
desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions'
- task generate: :environment do
+ task generate: ['gitlab:sidekiq:not_production', :environment] do
banner = <<~BANNER
# This file is generated automatically by
# bin/rake gitlab:sidekiq:all_queues_yml:generate
@@ -51,7 +46,7 @@ namespace :gitlab do
end
desc 'GitLab | Sidekiq | Validate that all_queues.yml matches worker definitions'
- task check: :environment do
+ task check: ['gitlab:sidekiq:not_production', :environment] do
if Gitlab::SidekiqConfig.all_queues_yml_outdated?
raise <<~MSG
Changes in worker queues found, please update the metadata by running:
@@ -70,7 +65,7 @@ namespace :gitlab do
namespace :sidekiq_queues_yml do
desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions'
- task generate: :environment do
+ task generate: ['gitlab:sidekiq:not_production', :environment] do
banner = <<~BANNER
# This file is generated automatically by
# bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate
@@ -104,7 +99,7 @@ namespace :gitlab do
end
desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions'
- task check: :environment do
+ task check: ['gitlab:sidekiq:not_production', :environment] do
if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated?
raise <<~MSG
Changes in worker queues found, please update the metadata by running:
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
index 0ad50c0fa53..166f08ef16a 100644
--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -19,7 +19,7 @@ namespace :gitlab do
desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application'
task generate_and_send: :environment do
- result = SubmitUsagePingService.new.execute
+ result = ServicePing::SubmitService.new.execute
puts Gitlab::Json.pretty_generate(result.attributes)
end