diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 09:08:42 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 09:08:42 +0000 |
commit | b76ae638462ab0f673e5915986070518dd3f9ad3 (patch) | |
tree | bdab0533383b52873be0ec0eb4d3c66598ff8b91 /lib/tasks | |
parent | 434373eabe7b4be9593d18a585fb763f1e5f1a6f (diff) | |
download | gitlab-ce-b76ae638462ab0f673e5915986070518dd3f9ad3.tar.gz |
Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/gitlab/backup.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/db.rake | 10 | ||||
-rw-r--r-- | lib/tasks/gitlab/docs/redirect.rake | 63 | ||||
-rw-r--r-- | lib/tasks/gitlab/gitaly.rake | 55 | ||||
-rw-r--r-- | lib/tasks/gitlab/graphql.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/info.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/product_intelligence.rake | 24 | ||||
-rw-r--r-- | lib/tasks/gitlab/smtp.rake | 23 | ||||
-rw-r--r-- | lib/tasks/gitlab/storage.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/usage_data.rake | 6 |
10 files changed, 118 insertions, 77 deletions
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index ed74dd472ff..cc10d73f76a 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -91,6 +91,8 @@ namespace :gitlab do backup.cleanup end + backup.remove_tmp + puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need to restore these files manually.".color(:red) puts "Restore task is done." @@ -297,7 +299,7 @@ namespace :gitlab do end def repository_backup_strategy - if Feature.enabled?(:gitaly_backup) + if Feature.enabled?(:gitaly_backup, default_enabled: :yaml) max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency) diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 2b508b341dd..51f15f5a56a 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -161,7 +161,7 @@ namespace :gitlab do exit end - indexes = Gitlab::Database::Reindexing.candidate_indexes + indexes = Gitlab::Database::PostgresIndex.reindexing_support if identifier = args[:index_name] raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ @@ -173,6 +173,12 @@ namespace :gitlab do ActiveRecord::Base.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) + # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any) + Gitlab::Database::Reindexing.cleanup_leftovers! + + # Hack: Before we do actual reindexing work, create async indexes + Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) + Gitlab::Database::Reindexing.perform(indexes) rescue StandardError => e Gitlab::AppLogger.error(e) @@ -217,7 +223,7 @@ namespace :gitlab do instrumentation = Gitlab::Database::Migrations::Instrumentation.new pending_migrations.each do |migration| - instrumentation.observe(migration.version) do + instrumentation.observe(version: migration.version, name: migration.name) do ActiveRecord::Migrator.new(:up, ctx.migrations, ctx.schema_migration, migration.version).run end end diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index 990ff723eeb..123a4775605 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -57,68 +57,5 @@ namespace :gitlab do post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->" end end - - desc 'GitLab | Docs | Clean up old redirects' - task :clean_redirects do - # - # Calculate new path from the redirect URL. - # - # If the redirect is not a full URL: - # 1. Create a new Pathname of the file - # 2. Use dirname to get all but the last component of the path - # 3. Join with the redirect_to entry - # 4. Substitute: - # - '.md' => '.html' - # - 'doc/' => '/ee/' - # - # If the redirect URL is a full URL pointing to the Docs site - # (cross-linking among the 4 products), remove the FQDN prefix: - # - # From : https://docs.gitlab.com/ee/install/requirements.html - # To : /ee/install/requirements.html - # - def new_path(redirect, filename) - if !redirect.start_with?('http') - Pathname.new(filename).dirname.join(redirect).to_s.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/') - elsif redirect.start_with?('https://docs.gitlab.com') - redirect.gsub('https://docs.gitlab.com', '') - else - redirect - end - end - - today = Time.now.utc.to_date - - # - # Find the files to be deleted. - # Exclude 'doc/development/documentation/index.md' because it - # contains an example of the YAML front matter. - # - files_to_be_deleted = `grep -Ir 'remove_date:' doc | grep -v doc/development/documentation/index.md | cut -d ":" -f 1`.split("\n") - - # - # Iterate over the files to be deleted and print the needed - # YAML entries for the Docs site redirects. - # - files_to_be_deleted.each do |filename| - frontmatter = YAML.safe_load(File.read(filename)) - remove_date = Date.parse(frontmatter['remove_date']) - old_path = filename.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/') - - # - # Check if the removal date is before today, and delete the file and - # print the content to be pasted in - # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/content/_data/redirects.yaml. - # The remove_date of redirects.yaml should be nine months in the future. - # To not be confused with the remove_date of the Markdown page. - # - next unless remove_date < today - - File.delete(filename) if File.exist?(filename) - puts " - from: #{old_path}" - puts " to: #{new_path(frontmatter['redirect_to'], filename)}" - puts " remove_date: #{remove_date >> 9}" - end - end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index df75b3cf716..6675439e430 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -2,6 +2,42 @@ namespace :gitlab do namespace :gitaly do + desc 'Installs gitaly for running tests within gitlab-development-kit' + task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| + inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT')) + + if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk + Rake::Task["gitlab:gitaly:install"].invoke(*args) + + next + end + + gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly')) + + # Our test setup expects a git repo, so clone rather than copy + version = Gitlab::GitalyClient.expected_server_version + checkout_or_clone_version(version: version, repo: gdk_gitaly_dir, target_dir: args.dir, clone_opts: %w[--depth 1]) + + # We assume the GDK gitaly already compiled binaries + build_dir = File.join(gdk_gitaly_dir, '_build') + FileUtils.cp_r(build_dir, args.dir) + + # We assume the GDK gitaly already ran bundle install + bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle') + FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby')) + + # For completeness we copy this for gitaly's make target + ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle') + FileUtils.cp_r(ruby_bundle_file, args.dir) + + gitaly_binary = File.join(build_dir, 'bin', 'gitaly') + warn_gitaly_out_of_date!(gitaly_binary, version) + rescue Errno::ENOENT => e + puts "Could not copy files, did you run `gdk update`? Error: #{e.message}" + + raise + end + desc 'GitLab | Gitaly | Install or upgrade gitaly' task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab @@ -41,5 +77,24 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") _, status = Gitlab::Popen.popen(%w[which gmake]) status == 0 ? 'gmake' : 'make' end + + def warn_gitaly_out_of_date!(gitaly_binary, expected_version) + binary_version, exit_status = Gitlab::Popen.popen(%W[#{gitaly_binary} -version]) + + raise "Failed to run `#{gitaly_binary} -version`" unless exit_status == 0 + + binary_version = binary_version.strip + + # See help for `git describe` for format + git_describe_sha = /g([a-f0-9]{5,40})\z/ + match = binary_version.match(git_describe_sha) + + # Just skip if the version does not have a sha component + return unless match + + return if expected_version.start_with?(match[1]) + + puts "WARNING: #{binary_version.strip} does not exactly match repository version #{expected_version}" + end end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index b405cbd3f68..52c5c680292 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -56,9 +56,9 @@ namespace :gitlab do color = case complexity when 0..GitlabSchema::DEFAULT_MAX_COMPLEXITY :green - when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_COMPLEXITY + when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY :yellow - when GitlabSchema::AUTHENTICATED_COMPLEXITY..GitlabSchema::ADMIN_COMPLEXITY + when GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY..GitlabSchema::ADMIN_MAX_COMPLEXITY :orange else :red diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 2826002bdc2..68395d10d24 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -68,8 +68,8 @@ namespace :gitlab do puts "Version:\t#{Gitlab::VERSION}" puts "Revision:\t#{Gitlab.revision}" puts "Directory:\t#{Rails.root}" - puts "DB Adapter:\t#{Gitlab::Database.human_adapter_name}" - puts "DB Version:\t#{Gitlab::Database.version}" + puts "DB Adapter:\t#{Gitlab::Database.main.human_adapter_name}" + puts "DB Version:\t#{Gitlab::Database.main.version}" puts "URL:\t\t#{Gitlab.config.gitlab.url}" puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" diff --git a/lib/tasks/gitlab/product_intelligence.rake b/lib/tasks/gitlab/product_intelligence.rake new file mode 100644 index 00000000000..329cd9c8c2a --- /dev/null +++ b/lib/tasks/gitlab/product_intelligence.rake @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :product_intelligence do + # @example + # bundle exec rake gitlab:product_intelligence:activate_metrics MILESTONE=14.0 + + desc 'GitLab | Product Intelligence | Update milestone metrics status to data_available' + task activate_metrics: :environment do + milestone = ENV['MILESTONE'] + raise "Please supply the MILESTONE env var".color(:red) unless milestone.present? + + Gitlab::Usage::MetricDefinition.definitions.values.each do |metric| + next if metric.attributes[:milestone] != milestone || metric.attributes[:status] != 'implemented' + + metric.attributes[:status] = 'data_available' + path = metric.path + File.open(path, "w") { |file| file << metric.to_h.deep_stringify_keys.to_yaml } + end + + puts "Task completed successfully" + end + end +end diff --git a/lib/tasks/gitlab/smtp.rake b/lib/tasks/gitlab/smtp.rake new file mode 100644 index 00000000000..23ad7577e3c --- /dev/null +++ b/lib/tasks/gitlab/smtp.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :smtp do + namespace :secret do + desc 'GitLab | SMTP | Secret | Write SMTP secrets' + task write: [:environment] do + content = $stdin.tty? ? $stdin.gets : $stdin.read + Gitlab::EncryptedSmtpCommand.write(content) + end + + desc 'GitLab | SMTP | Secret | Edit SMTP secrets' + task edit: [:environment] do + Gitlab::EncryptedSmtpCommand.edit + end + + desc 'GitLab | SMTP | Secret | Show SMTP secrets' + task show: [:environment] do + Gitlab::EncryptedSmtpCommand.show + end + end + end +end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 6fa39a26488..fb9f9b9fe67 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -170,7 +170,7 @@ namespace :gitlab do inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f attempts.to_i.times do - unless Gitlab::Database.exists? + unless Gitlab::Database.main.exists? puts "Waiting until database is ready before continuing...".color(:yellow) sleep inverval end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 166f08ef16a..ddd3424acda 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -24,12 +24,6 @@ namespace :gitlab do puts Gitlab::Json.pretty_generate(result.attributes) end - desc 'GitLab | UsageData | Generate metrics dictionary' - task generate_metrics_dictionary: :environment do - items = Gitlab::Usage::MetricDefinition.definitions - Gitlab::Usage::Docs::Renderer.new(items).write - end - desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON' task generate_from_yaml: :environment do puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data) |