diff options
Diffstat (limited to 'lib/tasks/gitlab')
-rw-r--r-- | lib/tasks/gitlab/artifacts/migrate.rake | 18 | ||||
-rw-r--r-- | lib/tasks/gitlab/db.rake | 26 | ||||
-rw-r--r-- | lib/tasks/gitlab/git.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/lfs/migrate.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/packages/events.rake | 6 | ||||
-rw-r--r-- | lib/tasks/gitlab/packages/migrate.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/pages.rake | 10 | ||||
-rw-r--r-- | lib/tasks/gitlab/praefect.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/sidekiq.rake | 23 | ||||
-rw-r--r-- | lib/tasks/gitlab/snowplow.rake | 11 | ||||
-rw-r--r-- | lib/tasks/gitlab/terraform/migrate.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/usage_data.rake | 5 |
12 files changed, 80 insertions, 33 deletions
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 94867e1a16a..4c312ea492b 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -8,30 +8,24 @@ namespace :gitlab do namespace :artifacts do task migrate: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts to remote storage') - helper = Gitlab::Artifacts::MigrationHelper.new + helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger) begin - helper.migrate_to_remote_storage do |artifact| - logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage") - end - rescue => e + helper.migrate_to_remote_storage + rescue StandardError => e logger.error(e.message) end end task migrate_to_local: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts to local storage') - helper = Gitlab::Artifacts::MigrationHelper.new + helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger) begin - helper.migrate_to_local_storage do |artifact| - logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage") - end - rescue => e + helper.migrate_to_local_storage + rescue StandardError => e logger.error(e.message) end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 3baf4e7b7c6..bbfdf598e42 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -191,7 +191,7 @@ namespace :gitlab do ActiveRecord::Base.logger = Logger.new(STDOUT) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) Gitlab::Database::Reindexing.perform(indexes) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error(e) raise end @@ -217,9 +217,11 @@ namespace :gitlab do end desc 'Run migrations with instrumentation' - task :migration_testing, [:result_file] => :environment do |_, args| - result_file = args[:result_file] || raise("Please specify result_file argument") - raise "File exists already, won't overwrite: #{result_file}" if File.exist?(result_file) + task migration_testing: :environment do + result_dir = Gitlab::Database::Migrations::Instrumentation::RESULT_DIR + raise "Directory exists already, won't overwrite: #{result_dir}" if File.exist?(result_dir) + + Dir.mkdir(result_dir) verbose_was = ActiveRecord::Migration.verbose ActiveRecord::Migration.verbose = true @@ -240,7 +242,7 @@ namespace :gitlab do end ensure if instrumentation - File.open(result_file, 'wb+') do |io| + File.open(File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME), 'wb+') do |io| io << instrumentation.observations.to_json end end @@ -248,5 +250,19 @@ namespace :gitlab do ActiveRecord::Base.clear_cache! ActiveRecord::Migration.verbose = verbose_was end + + desc 'Run all pending batched migrations' + task execute_batched_migrations: :environment do + Gitlab::Database::BackgroundMigration::BatchedMigration.active.queue_order.each do |migration| + Gitlab::AppLogger.info("Executing batched migration #{migration.id} inline") + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(migration) + end + end + + # Only for development environments, + # we execute pending data migrations inline for convenience. + Rake::Task['db:migrate'].enhance do + Rake::Task['gitlab:db:execute_batched_migrations'].invoke if Rails.env.development? + end end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 9c5549b4a54..8587ef4afdb 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -9,7 +9,7 @@ namespace :gitlab do begin project.repository.fsck - rescue => e + rescue StandardError => e failures << "#{project.full_path} on #{project.repository_storage}: #{e}" end @@ -51,7 +51,7 @@ namespace :gitlab do next unless project.repo_exists? result = project.repository.checksum - rescue => e + rescue StandardError => e result = "Ignored error: #{e.message}".squish.truncate(255) ensure puts "#{project.id},#{result}" diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 05249a126bc..a173de7c5c7 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -14,7 +14,7 @@ namespace :gitlab do lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") - rescue => e + rescue StandardError => e logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end @@ -28,7 +28,7 @@ namespace :gitlab do lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL) logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage") - rescue => e + rescue StandardError => e logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index 4a6a014acc5..d24535d85b6 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -9,7 +9,7 @@ namespace :gitlab do task generate: :environment do Rake::Task["gitlab:packages:events:generate_counts"].invoke Rake::Task["gitlab:packages:events:generate_unique"].invoke - rescue => e + rescue StandardError => e logger.error("Error building events list: #{e}") end @@ -21,7 +21,7 @@ namespace :gitlab do File.open(path, "w") { |file| file << counter_events_list.to_yaml } logger.info("Events file `#{path}` generated successfully") - rescue => e + rescue StandardError => e logger.error("Error building events list: #{e}") end @@ -33,7 +33,7 @@ namespace :gitlab do File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml } logger.info("Events file `#{path}` generated successfully") - rescue => e + rescue StandardError => e logger.error("Error building events list: #{e}") end diff --git a/lib/tasks/gitlab/packages/migrate.rake b/lib/tasks/gitlab/packages/migrate.rake index 20a8c51db66..febc3e7fa2d 100644 --- a/lib/tasks/gitlab/packages/migrate.rake +++ b/lib/tasks/gitlab/packages/migrate.rake @@ -17,7 +17,7 @@ namespace :gitlab do package_file.file.migrate!(::Packages::PackageFileUploader::Store::REMOTE) logger.info("Transferred package file #{package_file.id} of size #{package_file.size.to_i.bytes} to object storage") - rescue => e + rescue StandardError => e logger.error("Failed to transfer package file #{package_file.id} with error: #{e.message}") end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index ee2931f0c4f..684d62d1367 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -61,26 +61,24 @@ namespace :gitlab do namespace :deployments do task migrate_to_object_storage: :gitlab_environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of pages deployments to remote storage') - helper = Gitlab::Pages::MigrationHelper.new(logger) + helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger) begin helper.migrate_to_remote_storage - rescue => e + rescue StandardError => e logger.error(e.message) end end task migrate_to_local: :gitlab_environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of Pages deployments to local storage') - helper = Gitlab::Pages::MigrationHelper.new(logger) + helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger) begin helper.migrate_to_local_storage - rescue => e + rescue StandardError => e logger.error(e.message) end end diff --git a/lib/tasks/gitlab/praefect.rake b/lib/tasks/gitlab/praefect.rake index 346df3e0c75..28b70f8986e 100644 --- a/lib/tasks/gitlab/praefect.rake +++ b/lib/tasks/gitlab/praefect.rake @@ -44,7 +44,7 @@ namespace :gitlab do row = [project.name] << replicas_resp.primary.checksum row.concat(sorted_replicas.map {|r| r.checksum}) - rescue + rescue StandardError puts 'Something went wrong when getting replicas.' next end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 6f00db42d78..6f5c3a86dd3 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -8,6 +8,29 @@ namespace :gitlab do File.write(path, banner + YAML.dump(object).gsub(/ *$/m, '')) end + namespace :migrate_jobs do + def mappings + ::Gitlab::SidekiqConfig + .workers + .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) } + .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] } + end + + desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names' + task schedule: :environment do + ::Gitlab::SidekiqMigrateJobs + .new('schedule', logger: Logger.new($stdout)) + .execute(mappings) + end + + desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names' + task retry: :environment do + ::Gitlab::SidekiqMigrateJobs + .new('retry', logger: Logger.new($stdout)) + .execute(mappings) + end + end + namespace :all_queues_yml do desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' task generate: :environment do diff --git a/lib/tasks/gitlab/snowplow.rake b/lib/tasks/gitlab/snowplow.rake new file mode 100644 index 00000000000..278ba4a471c --- /dev/null +++ b/lib/tasks/gitlab/snowplow.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :snowplow do + desc 'GitLab | Snowplow | Generate event dictionary' + task generate_event_dictionary: :environment do + items = Gitlab::Tracking::EventDefinition.definitions + Gitlab::Tracking::Docs::Renderer.new(items).write + end + end +end diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake index a9c16049240..2bf9ec9537a 100644 --- a/lib/tasks/gitlab/terraform/migrate.rake +++ b/lib/tasks/gitlab/terraform/migrate.rake @@ -15,7 +15,7 @@ namespace :gitlab do logger.info(message) end - rescue => e + rescue StandardError => e logger.error("Failed to migrate: #{e.message}") end end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 95072444fcf..0ad50c0fa53 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -29,5 +29,10 @@ namespace :gitlab do items = Gitlab::Usage::MetricDefinition.definitions Gitlab::Usage::Docs::Renderer.new(items).write end + + desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON' + task generate_from_yaml: :environment do + puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data) + end end end |