From a7b3560714b4d9cc4ab32dffcd1f74a284b93580 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 18 Feb 2022 09:45:46 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-8-stable-ee --- lib/tasks/gitlab/assets.rake | 32 +-- lib/tasks/gitlab/backup.rake | 284 ++---------------------- lib/tasks/gitlab/banzai.rake | 20 ++ lib/tasks/gitlab/db.rake | 44 +++- lib/tasks/gitlab/dependency_proxy/migrate.rake | 33 +++ lib/tasks/gitlab/docs/compile_deprecations.rake | 32 ++- lib/tasks/gitlab/info.rake | 4 - lib/tasks/gitlab/tw/codeowners.rake | 111 +++++++++ lib/tasks/gitlab/usage_data.rake | 6 +- 9 files changed, 262 insertions(+), 304 deletions(-) create mode 100644 lib/tasks/gitlab/banzai.rake create mode 100644 lib/tasks/gitlab/dependency_proxy/migrate.rake create mode 100644 lib/tasks/gitlab/tw/codeowners.rake (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index cbafed16852..0b70dba5c05 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -10,20 +10,20 @@ module Tasks JH_ASSET_FOLDERS = %w[jh/app/assets].freeze JS_ASSET_PATTERNS = %w[*.js config/**/*.js].freeze JS_ASSET_FILES = %w[package.json yarn.lock].freeze - MASTER_MD5_HASH_FILE = 'master-assets-hash.txt' - HEAD_MD5_HASH_FILE = 'assets-hash.txt' + MASTER_SHA256_HASH_FILE = 'master-assets-hash.txt' + HEAD_SHA256_HASH_FILE = 'assets-hash.txt' PUBLIC_ASSETS_WEBPACK_DIR = 'public/assets/webpack' - def self.md5_of_assets_impacting_webpack_compilation + def self.sha256_of_assets_impacting_webpack_compilation start_time = Time.now asset_files = assets_impacting_webpack_compilation - puts "Generating the MD5 hash for #{assets_impacting_webpack_compilation.size} Webpack-related assets..." + puts "Generating the SHA256 hash for #{assets_impacting_webpack_compilation.size} Webpack-related assets..." - asset_file_md5s = asset_files.map do |asset_file| - Digest::MD5.file(asset_file).hexdigest + asset_file_sha256s = asset_files.map do |asset_file| + Digest::SHA256.file(asset_file).hexdigest end - Digest::MD5.hexdigest(asset_file_md5s.join).tap { |md5| puts "=> MD5 generated in #{Time.now - start_time}: #{md5}" } + Digest::SHA256.hexdigest(asset_file_sha256s.join).tap { |sha256| puts "=> SHA256 generated in #{Time.now - start_time}: #{sha256}" } end def self.assets_impacting_webpack_compilation @@ -63,25 +63,25 @@ namespace :gitlab do desc 'GitLab | Assets | Compile all Webpack assets' task :compile_webpack_if_needed do - FileUtils.mv(Tasks::Gitlab::Assets::HEAD_MD5_HASH_FILE, Tasks::Gitlab::Assets::MASTER_MD5_HASH_FILE, force: true) + FileUtils.mv(Tasks::Gitlab::Assets::HEAD_SHA256_HASH_FILE, Tasks::Gitlab::Assets::MASTER_SHA256_HASH_FILE, force: true) - master_assets_md5 = - if File.exist?(Tasks::Gitlab::Assets::MASTER_MD5_HASH_FILE) - File.read(Tasks::Gitlab::Assets::MASTER_MD5_HASH_FILE) + master_assets_sha256 = + if File.exist?(Tasks::Gitlab::Assets::MASTER_SHA256_HASH_FILE) + File.read(Tasks::Gitlab::Assets::MASTER_SHA256_HASH_FILE) else 'missing!' end - head_assets_md5 = Tasks::Gitlab::Assets.md5_of_assets_impacting_webpack_compilation.tap do |md5| - File.write(Tasks::Gitlab::Assets::HEAD_MD5_HASH_FILE, md5) + head_assets_sha256 = Tasks::Gitlab::Assets.sha256_of_assets_impacting_webpack_compilation.tap do |sha256| + File.write(Tasks::Gitlab::Assets::HEAD_SHA256_HASH_FILE, sha256) end - puts "Webpack assets MD5 for `master`: #{master_assets_md5}" - puts "Webpack assets MD5 for `HEAD`: #{head_assets_md5}" + puts "Webpack assets SHA256 for `master`: #{master_assets_sha256}" + puts "Webpack assets SHA256 for `HEAD`: #{head_assets_sha256}" public_assets_webpack_dir_exists = Dir.exist?(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) - if head_assets_md5 != master_assets_md5 || !public_assets_webpack_dir_exists + if head_assets_sha256 != master_assets_sha256 || !public_assets_webpack_dir_exists FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists unless system('yarn webpack') diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 0bca63a64f5..78cb7d72d4f 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -9,26 +9,7 @@ namespace :gitlab do task create: :gitlab_environment do warn_user_is_not_gitlab - %w(db repo uploads builds artifacts pages lfs terraform_state registry packages).each do |type| - Rake::Task["gitlab:backup:#{type}:create"].invoke - end - - backup = Backup::Manager.new(progress) - backup.write_info - - if ENV['SKIP'] && ENV['SKIP'].include?('tar') - backup.upload - else - backup.pack - backup.upload - backup.cleanup - backup.remove_old - end - - progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ - "and are not included in this backup. You will need these files to restore a backup.\n" \ - "Please back them up manually.".color(:red) - progress.puts "Backup task is done." + Backup::Manager.new(progress).create end # Restore backup of GitLab system @@ -36,320 +17,109 @@ namespace :gitlab do task restore: :gitlab_environment do warn_user_is_not_gitlab - backup = Backup::Manager.new(progress) - cleanup_required = backup.unpack - backup.verify_backup_version - - unless backup.skipped?('db') - begin - unless ENV['force'] == 'yes' - warning = <<-MSG.strip_heredoc - Be sure to stop Puma, Sidekiq, and any other process that - connects to the database before proceeding. For Omnibus - installs, see the following link for more information: - https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations - - Before restoring the database, we will remove all existing - tables to avoid future upgrade problems. Be aware that if you have - custom tables in the GitLab database these tables and all data will be - removed. - MSG - puts warning.color(:red) - ask_to_continue - puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) - sleep(5) - end - - # Drop all tables Load the schema to ensure we don't have any newer tables - # hanging out from a failed upgrade - puts_time 'Cleaning the database ... '.color(:blue) - Rake::Task['gitlab:db:drop_tables'].invoke - puts_time 'done'.color(:green) - Rake::Task['gitlab:backup:db:restore'].invoke - rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".color(:red) - exit 1 - end - end - - Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') - Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') - Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') - Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') - Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages') - Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') - Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state') - Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') - Rake::Task['gitlab:backup:packages:restore'].invoke unless backup.skipped?('packages') - Rake::Task['gitlab:shell:setup'].invoke - Rake::Task['cache:clear'].invoke - - if cleanup_required - backup.cleanup - end - - backup.remove_tmp - - puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ - "and are not included in this backup. You will need to restore these files manually.".color(:red) - puts "Restore task is done." + Backup::Manager.new(progress).restore end namespace :repo do task create: :gitlab_environment do - puts_time "Dumping repositories ...".color(:blue) - - max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i - max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i - - if ENV["SKIP"] && ENV["SKIP"].include?("repositories") - puts_time "[SKIPPED]".color(:cyan) - elsif max_concurrency < 1 || max_storage_concurrency < 1 - puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) - exit 1 - else - Backup::Repositories.new(progress, strategy: repository_backup_strategy).dump( - max_concurrency: max_concurrency, - max_storage_concurrency: max_storage_concurrency - ) - puts_time "done".color(:green) - end + Backup::Manager.new(progress).run_create_task('repositories') end task restore: :gitlab_environment do - puts_time "Restoring repositories ...".color(:blue) - Backup::Repositories.new(progress, strategy: repository_backup_strategy).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('repositories') end end namespace :db do task create: :gitlab_environment do - puts_time "Dumping database ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("db") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Database.new(progress).dump - puts_time "done".color(:green) - rescue Backup::DatabaseBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('db') end task restore: :gitlab_environment do - puts_time "Restoring database ... ".color(:blue) - errors = Backup::Database.new(progress).restore - - if errors.present? - warning = <<~MSG - There were errors in restoring the schema. This may cause - issues if this results in missing indexes, constraints, or - columns. Please record the errors above and contact GitLab - Support if you have questions: - https://about.gitlab.com/support/ - MSG - - warn warning.color(:red) - ask_to_continue - end - - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('db') end end namespace :builds do task create: :gitlab_environment do - puts_time "Dumping builds ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("builds") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Builds.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('builds') end task restore: :gitlab_environment do - puts_time "Restoring builds ... ".color(:blue) - Backup::Builds.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('builds') end end namespace :uploads do task create: :gitlab_environment do - puts_time "Dumping uploads ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("uploads") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Uploads.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('uploads') end task restore: :gitlab_environment do - puts_time "Restoring uploads ... ".color(:blue) - Backup::Uploads.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('uploads') end end namespace :artifacts do task create: :gitlab_environment do - puts_time "Dumping artifacts ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Artifacts.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('artifacts') end task restore: :gitlab_environment do - puts_time "Restoring artifacts ... ".color(:blue) - Backup::Artifacts.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('artifacts') end end namespace :pages do task create: :gitlab_environment do - puts_time "Dumping pages ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("pages") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Pages.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('pages') end task restore: :gitlab_environment do - puts_time "Restoring pages ... ".color(:blue) - Backup::Pages.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('pages') end end namespace :lfs do task create: :gitlab_environment do - puts_time "Dumping lfs objects ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("lfs") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Lfs.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('lfs') end task restore: :gitlab_environment do - puts_time "Restoring lfs objects ... ".color(:blue) - Backup::Lfs.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('lfs') end end namespace :terraform_state do task create: :gitlab_environment do - puts_time "Dumping terraform states ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state") - puts_time "[SKIPPED]".color(:cyan) - else - Backup::TerraformState.new(progress).dump - puts_time "done".color(:green) - end + Backup::Manager.new(progress).run_create_task('terraform_state') end task restore: :gitlab_environment do - puts_time "Restoring terraform states ... ".color(:blue) - Backup::TerraformState.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('terraform_state') end end namespace :registry do task create: :gitlab_environment do - puts_time "Dumping container registry images ... ".color(:blue) - - if Gitlab.config.registry.enabled - if ENV["SKIP"] && ENV["SKIP"].include?("registry") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Registry.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end - else - puts_time "[DISABLED]".color(:cyan) - end + Backup::Manager.new(progress).run_create_task('registry') end task restore: :gitlab_environment do - puts_time "Restoring container registry images ... ".color(:blue) - - if Gitlab.config.registry.enabled - Backup::Registry.new(progress).restore - puts_time "done".color(:green) - else - puts_time "[DISABLED]".color(:cyan) - end + Backup::Manager.new(progress).run_restore_task('registry') end end namespace :packages do task create: :gitlab_environment do - puts_time "Dumping packages ... ".color(:blue) - - if ENV['SKIP'] && ENV['SKIP'].include?('packages') - puts_time "[SKIPPED]".color(:cyan) - else - Backup::Packages.new(progress).dump - puts_time "done".color(:green) - end + Backup::Manager.new(progress).run_create_task('packages') end task restore: :gitlab_environment do - puts_time "Restoring packages ...".color(:blue) - Backup::Packages.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('packages') end end - def puts_time(msg) - progress.puts "#{Time.now} -- #{msg}" - Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") - end - def progress if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a @@ -360,16 +130,6 @@ namespace :gitlab do $stdout end end - - def repository_backup_strategy - if Feature.enabled?(:gitaly_backup, default_enabled: :yaml) - max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence - max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence - Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) - else - Backup::GitalyRpcBackup.new(progress) - end - end end # namespace end: backup end diff --git a/lib/tasks/gitlab/banzai.rake b/lib/tasks/gitlab/banzai.rake new file mode 100644 index 00000000000..b1c7e4ea519 --- /dev/null +++ b/lib/tasks/gitlab/banzai.rake @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :banzai do + desc 'GitLab | Banzai | Render markdown using our FullPipeline (input will be requested)' + task render: :environment do |_t| + markdown = [] + + puts "\nEnter markdown below, Ctrl-D to end (if you need blank lines, paste in the full text):" + while buf = Readline.readline('', true) + markdown << buf + end + + puts "Rendering using Gitlab's FullPipeline...\n\n" + + html = MarkupHelper.markdown(markdown.join("\n"), { pipeline: :full, project: nil }) + puts html.gsub(' ', "\n") + end + end +end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index efb0e1ef1e1..6d4af9d166f 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -6,23 +6,32 @@ namespace :gitlab do namespace :db do desc 'GitLab | DB | Manually insert schema migration version' task :mark_migration_complete, [:version] => :environment do |_, args| - unless args[:version] - puts "Must specify a migration version as an argument".color(:red) - exit 1 + mark_migration_complete(args[:version]) + end + + namespace :mark_migration_complete do + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + desc "Gitlab | DB | Manually insert schema migration version on #{name} database" + task name, [:version] => :environment do |_, args| + mark_migration_complete(args[:version], database: name) + end end + end - version = args[:version].to_i - if version == 0 - puts "Version '#{args[:version]}' must be a non-zero integer".color(:red) + def mark_migration_complete(version, database: nil) + if version.to_i == 0 + puts 'Must give a version argument that is a non-zero integer'.color(:red) exit 1 end - sql = "INSERT INTO schema_migrations (version) VALUES (#{version})" - begin - ActiveRecord::Base.connection.execute(sql) - puts "Successfully marked '#{version}' as complete".color(:green) + Gitlab::Database.database_base_models.each do |name, model| + next if database && database.to_s != name + + model.connection.execute("INSERT INTO schema_migrations (version) VALUES (#{model.connection.quote(version)})") + + puts "Successfully marked '#{version}' as complete on database #{name}".color(:green) rescue ActiveRecord::RecordNotUnique - puts "Migration version '#{version}' is already marked complete".color(:yellow) + puts "Migration version '#{version}' is already marked complete on database #{name}".color(:yellow) end end @@ -261,6 +270,19 @@ namespace :gitlab do end end + desc 'Run migration as gitlab non-superuser' + task :reset_as_non_superuser, [:username] => :environment do |_, args| + username = args.fetch(:username, 'gitlab') + puts "Migrate using username #{username}" + Rake::Task['db:drop'].invoke + Rake::Task['db:create'].invoke + ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| + ActiveRecord::Base.establish_connection(db_config.configuration_hash.merge(username: username)) # rubocop: disable Database/EstablishConnection + Gitlab::Database.check_for_non_superuser + Rake::Task['db:migrate'].invoke + end + end + # Only for development environments, # we execute pending data migrations inline for convenience. Rake::Task['db:migrate'].enhance do diff --git a/lib/tasks/gitlab/dependency_proxy/migrate.rake b/lib/tasks/gitlab/dependency_proxy/migrate.rake new file mode 100644 index 00000000000..fa35eacc59d --- /dev/null +++ b/lib/tasks/gitlab/dependency_proxy/migrate.rake @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'logger' + +desc "GitLab | Dependency Proxy | Migrate dependency proxy files to remote storage" +namespace :gitlab do + namespace :dependency_proxy do + task migrate: :environment do + logger = Logger.new($stdout) + logger.info('Starting transfer of dependency proxy files to object storage') + + unless ::DependencyProxy::FileUploader.object_store_enabled? + raise 'Object store is disabled for dependency proxy feature' + end + + ::DependencyProxy::Blob.with_files_stored_locally.find_each(batch_size: 10) do |blob_file| + blob_file.file.migrate!(::DependencyProxy::FileUploader::Store::REMOTE) + + logger.info("Transferred dependency proxy blob file #{blob_file.id} of size #{blob_file.size.to_i.bytes} to object storage") + rescue StandardError => e + logger.error("Failed to transfer dependency proxy blob file #{blob_file.id} with error: #{e.message}") + end + + ::DependencyProxy::Manifest.with_files_stored_locally.find_each(batch_size: 10) do |manifest_file| + manifest_file.file.migrate!(::DependencyProxy::FileUploader::Store::REMOTE) + + logger.info("Transferred dependency proxy manifest file #{manifest_file.id} of size #{manifest_file.size.to_i.bytes} to object storage") + rescue StandardError => e + logger.error("Failed to transfer dependency proxy manifest file #{manifest_file.id} with error: #{e.message}") + end + end + end +end diff --git a/lib/tasks/gitlab/docs/compile_deprecations.rake b/lib/tasks/gitlab/docs/compile_deprecations.rake index 4ac68a9f850..f7821315f82 100644 --- a/lib/tasks/gitlab/docs/compile_deprecations.rake +++ b/lib/tasks/gitlab/docs/compile_deprecations.rake @@ -2,15 +2,19 @@ namespace :gitlab do namespace :docs do + COLOR_CODE_RESET = "\e[0m" + COLOR_CODE_RED = "\e[31m" + COLOR_CODE_GREEN = "\e[32m" + desc "Generate deprecation list from individual files" task :compile_deprecations do require_relative '../../../../tooling/docs/deprecation_handling' path = Rails.root.join("doc/update/deprecations.md") File.write(path, Docs::DeprecationHandling.new('deprecation').render) - puts "Deprecations compiled to #{path}" + puts "#{COLOR_CODE_GREEN}INFO: Deprecations compiled to #{path}.#{COLOR_CODE_RESET}" end - desc "Check that the deprecation doc is up to date" + desc "Check that the deprecation documentation is up to date" task :check_deprecations do require_relative '../../../../tooling/docs/deprecation_handling' path = Rails.root.join("doc/update/deprecations.md") @@ -19,9 +23,15 @@ namespace :gitlab do doc = File.read(path) if doc == contents - puts "Deprecations doc is up to date." + puts "#{COLOR_CODE_GREEN}INFO: Deprecations documentation is up to date.#{COLOR_CODE_RESET}" else - format_output('Deprecations doc is outdated! You (or your technical writer) can update it by running `bin/rake gitlab:docs:compile_deprecations`.') + warn <<~EOS + #{COLOR_CODE_RED}ERROR: Deprecations documentation is outdated!#{COLOR_CODE_RESET} + To update the deprecations documentation, either: + + - Run `bin/rake gitlab:docs:compile_deprecations` and commit the changes to this branch. + - Have a technical writer resolve the issue. + EOS abort end end @@ -31,10 +41,10 @@ namespace :gitlab do require_relative '../../../../tooling/docs/deprecation_handling' path = Rails.root.join("doc/update/removals.md") File.write(path, Docs::DeprecationHandling.new('removal').render) - puts "Removals compiled to #{path}" + puts "#{COLOR_CODE_GREEN}INFO: Removals compiled to #{path}.#{COLOR_CODE_RESET}" end - desc "Check that the removal doc is up to date" + desc "Check that the removal documentation is up to date" task :check_removals do require_relative '../../../../tooling/docs/deprecation_handling' path = Rails.root.join("doc/update/removals.md") @@ -42,9 +52,15 @@ namespace :gitlab do doc = File.read(path) if doc == contents - puts "Removals doc is up to date." + puts "#{COLOR_CODE_GREEN}INFO: Removals documentation is up to date.#{COLOR_CODE_RESET}" else - format_output('Removals doc is outdated! You (or your technical writer) can update it by running `bin/rake gitlab:docs:compile_removals`.') + warn <<~EOS + #{COLOR_CODE_RED}ERROR: Removals documentation is outdated!#{COLOR_CODE_RESET} + To update the removals documentation, either: + + - Run `bin/rake gitlab:docs:compile_removals` and commit the changes to this branch. + - Have a technical writer resolve the issue. + EOS abort end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 02764b5d46f..6f42bf8c946 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -22,8 +22,6 @@ namespace :gitlab do proxies = Gitlab::Proxy.detect_proxy.map {|k, v| "#{k}: #{v}"}.join("\n\t\t") end - # check Git version - git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a # check Go version go_version = run_and_match(%w(go version), /go version (.+)/).to_a @@ -43,7 +41,6 @@ namespace :gitlab do puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" - puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}" @@ -95,7 +92,6 @@ namespace :gitlab do end end puts "GitLab Shell path:\t\t#{Gitlab.config.gitlab_shell.path}" - puts "Git:\t\t#{Gitlab.config.git.bin_path}" end end end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake new file mode 100644 index 00000000000..43fd4f8685a --- /dev/null +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'yaml' + +namespace :tw do + desc 'Generates a list of codeowners for documentation pages.' + task :codeowners do + CodeOwnerRule = Struct.new(:category, :writer) + + CODE_OWNER_RULES = [ + CodeOwnerRule.new('Activation', '@kpaizee'), + CodeOwnerRule.new("Adoption", '@kpaizee'), + CodeOwnerRule.new('Activation', '@kpaizee'), + CodeOwnerRule.new('Adoption', '@kpaizee'), + CodeOwnerRule.new('APM', '@ngaskill'), + CodeOwnerRule.new('Authentication & Authorization', '@eread'), + CodeOwnerRule.new('Certify', '@msedlakjakubowski'), + CodeOwnerRule.new('Code Review', '@aqualls'), + CodeOwnerRule.new('Compliance', '@eread'), + CodeOwnerRule.new('Composition Analysis', '@rdickenson'), + CodeOwnerRule.new('Configure', '@marcia'), + CodeOwnerRule.new('Container Security', '@ngaskill'), + CodeOwnerRule.new('Contributor Experience', '@eread'), + CodeOwnerRule.new('Conversion', '@kpaizee'), + CodeOwnerRule.new('Database', '@marcia'), + CodeOwnerRule.new('Development', '@marcia'), + CodeOwnerRule.new('Distribution', '@axil'), + CodeOwnerRule.new('Distribution (Charts)', '@axil'), + CodeOwnerRule.new('Distribution (Omnibus)', '@axil'), + CodeOwnerRule.new('Documentation Guidelines', '@cnorris'), + CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), + CodeOwnerRule.new('Ecosystem', '@kpaizee'), + CodeOwnerRule.new('Editor', '@aqualls'), + CodeOwnerRule.new('Expansion', '@kpaizee'), + CodeOwnerRule.new('Foundations', '@rdickenson'), + CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), + CodeOwnerRule.new('Geo', '@axil'), + CodeOwnerRule.new('Gitaly', '@eread'), + CodeOwnerRule.new('Global Search', '@marcia'), + CodeOwnerRule.new('Health', '@ngaskill'), + CodeOwnerRule.new('Import', '@ngaskill'), + CodeOwnerRule.new('Infrastructure', '@marcia'), + CodeOwnerRule.new('Integrations', '@kpaizee'), + CodeOwnerRule.new('Knowledge', '@aqualls'), + CodeOwnerRule.new('License', '@sselhorn'), + CodeOwnerRule.new('Memory', '@marcia'), + CodeOwnerRule.new('Monitor', '@ngaskill'), + CodeOwnerRule.new('Optimize', '@fneill'), + CodeOwnerRule.new('Package', '@ngaskill'), + CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault'), + CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault'), + CodeOwnerRule.new('Portfolio Management', '@msedlakjakubowski'), + CodeOwnerRule.new('Product Intelligence', '@fneill'), + CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'), + CodeOwnerRule.new('Project Management', '@msedlakjakubowski'), + CodeOwnerRule.new('Provision', '@sselhorn'), + CodeOwnerRule.new('Purchase', '@sselhorn'), + CodeOwnerRule.new('Redirect', 'Redirect'), + CodeOwnerRule.new('Release', '@rdickenson'), + CodeOwnerRule.new('Runner', '@sselhorn'), + CodeOwnerRule.new('Sharding', '@marcia'), + CodeOwnerRule.new('Source Code', '@aqualls'), + CodeOwnerRule.new('Static Analysis', '@rdickenson'), + CodeOwnerRule.new('Static Site Editor', '@aqualls'), + CodeOwnerRule.new('Style Guide', '@sselhorn'), + CodeOwnerRule.new('Testing', '@eread'), + CodeOwnerRule.new('Threat Insights', '@fneill'), + CodeOwnerRule.new('Utilization', '@sselhorn'), + CodeOwnerRule.new('Vulnerability Research', '@fneill'), + CodeOwnerRule.new('Workspace', '@fneill') + ].freeze + + Document = Struct.new(:group, :redirect) do + def has_a_valid_group? + group && !redirect + end + + def missing_metadata? + !group && !redirect + end + end + + def self.writer_for_group(category) + CODE_OWNER_RULES.find { |rule| rule.category == category }&.writer + end + + errors = [] + + path = Rails.root.join("doc/**/*.md") + Dir.glob(path) do |file| + yaml_data = YAML.load_file(file) + document = Document.new(yaml_data['group'], yaml_data['redirect_to']) + + if document.missing_metadata? + errors << file + next + end + + writer = writer_for_group(document.group) + next unless writer + + puts "#{file.gsub(Dir.pwd, ".")} #{writer}" if document.has_a_valid_group? + end + + if errors.present? + puts "-----" + puts "ERRORS - the following files are missing the correct metadata:" + errors.map { |file| puts file.gsub(Dir.pwd, ".")} + end + end +end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 694c49240ed..9f064ef4c0c 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -4,17 +4,17 @@ namespace :gitlab do namespace :usage_data do desc 'GitLab | UsageData | Generate raw SQLs for usage ping in YAML' task dump_sql_in_yaml: :environment do - puts Gitlab::UsageDataQueries.uncached_data.to_yaml + puts Gitlab::Usage::ServicePingReport.for(output: :metrics_queries).to_yaml end desc 'GitLab | UsageData | Generate raw SQLs for usage ping in JSON' task dump_sql_in_json: :environment do - puts Gitlab::Json.pretty_generate(Gitlab::UsageDataQueries.uncached_data) + puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :metrics_queries)) end desc 'GitLab | UsageData | Generate usage ping in JSON' task generate: :environment do - puts Gitlab::Json.pretty_generate(Gitlab::UsageData.uncached_data) + puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)) end desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application' -- cgit v1.2.1