diff options
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/frontend.rake | 5 | ||||
-rw-r--r-- | lib/tasks/gitlab/artifacts/migrate.rake | 29 | ||||
-rw-r--r-- | lib/tasks/gitlab/cleanup.rake | 63 | ||||
-rw-r--r-- | lib/tasks/gitlab/graphql.rake | 20 | ||||
-rw-r--r-- | lib/tasks/gitlab/lfs/migrate.rake | 15 | ||||
-rw-r--r-- | lib/tasks/gitlab/pages.rake | 9 | ||||
-rw-r--r-- | lib/tasks/gitlab/setup.rake | 1 | ||||
-rw-r--r-- | lib/tasks/gitlab/traces.rake | 38 | ||||
-rw-r--r-- | lib/tasks/gitlab/uploads/migrate.rake | 51 | ||||
-rw-r--r-- | lib/tasks/migrate/setup_postgresql.rake | 11 | ||||
-rw-r--r-- | lib/tasks/services.rake | 98 |
11 files changed, 73 insertions, 267 deletions
diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake index 1cac7520227..6e90229830d 100644 --- a/lib/tasks/frontend.rake +++ b/lib/tasks/frontend.rake @@ -2,7 +2,10 @@ unless Rails.env.production? namespace :frontend do desc 'GitLab | Frontend | Generate fixtures for JavaScript tests' RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| - args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb') + directories = %w[spec] + directories << 'ee/spec' if Gitlab.ee? + directory_glob = "{#{directories.join(',')}}" + args.with_defaults(pattern: "#{directory_glob}/frontend/fixtures/*.rb") ENV['NO_KNAPSACK'] = 'true' t.pattern = args[:pattern] t.rspec_opts = '--format documentation' diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 9012e55a70c..0d09fd0a4e3 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -6,18 +6,31 @@ namespace :gitlab do namespace :artifacts do task migrate: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts') + logger.info('Starting transfer of artifacts to remote storage') - Ci::Build.joins(:project) - .with_artifacts_stored_locally - .find_each(batch_size: 10) do |build| + helper = Gitlab::Artifacts::MigrationHelper.new - build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) - build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) + begin + helper.migrate_to_remote_storage do |artifact| + logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage") + end + rescue => e + logger.error(e.message) + end + end + + task migrate_to_local: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of artifacts to local storage') + + helper = Gitlab::Artifacts::MigrationHelper.new - logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") + begin + helper.migrate_to_local_storage do |artifact| + logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage") + end rescue => e - logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + logger.error(e.message) end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 4d854cd178d..0a0ee7b4bfa 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -3,69 +3,6 @@ require 'set' namespace :gitlab do namespace :cleanup do - desc "GitLab | Cleanup | Clean namespaces" - task dirs: :gitlab_environment do - namespaces = Set.new(Namespace.pluck(:path)) - namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX - - Gitaly::Server.all.each do |server| - all_dirs = Gitlab::GitalyClient::StorageService - .new(server.storage) - .list_directories(depth: 0) - .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) } - - puts "Looking for directories to remove... " - all_dirs.each do |dir_path| - if remove? - begin - Gitlab::GitalyClient::NamespaceService.new(server.storage) - .remove(dir_path) - - puts "Removed...#{dir_path}" - rescue StandardError => e - puts "Cannot remove #{dir_path}: #{e.message}".color(:red) - end - else - puts "Can be removed: #{dir_path}".color(:red) - end - end - end - - unless remove? - puts "To cleanup this directories run this command with REMOVE=true".color(:yellow) - end - end - - desc "GitLab | Cleanup | Clean repositories" - task repos: :gitlab_environment do - move_suffix = "+orphaned+#{Time.now.to_i}" - - Gitaly::Server.all.each do |server| - Gitlab::GitalyClient::StorageService - .new(server.storage) - .list_directories - .each do |path| - repo_with_namespace = path.chomp('.git').chomp('.wiki') - - # TODO ignoring hashed repositories for now. But revisit to fully support - # possible orphaned hashed repos - next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX) - next if Project.find_by_full_path(repo_with_namespace) - - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - - begin - Gitlab::GitalyClient::NamespaceService - .new(server.storage) - .rename(path, new_path) - rescue StandardError => e - puts "Error occurred while moving the repository: #{e.message}".color(:red) - end - end - end - end - desc "GitLab | Cleanup | Block users that have been removed in LDAP" task block_removed_ldap_users: :gitlab_environment do warn_user_is_not_gitlab diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index fd8df015903..902f22684ee 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -11,10 +11,28 @@ namespace :gitlab do task compile_docs: :environment do renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) - renderer.render + renderer.write puts "Documentation compiled." end + + desc 'GitLab | Check if GraphQL docs are up to date' + task check_docs: :environment do + renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) + + doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md')) + + if doc == renderer.contents + puts "GraphQL documentation is up to date" + else + puts '#' * 10 + puts '#' + puts '# GraphQL documentation is outdated! Please update it by running `bundle exec rake gitlab:graphql:compile_docs`.' + puts '#' + puts '#' * 10 + abort + end + end end end diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 97c15175a23..4142903d9c3 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -17,5 +17,20 @@ namespace :gitlab do logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end + + task migrate_to_local: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of LFS files to local storage') + + LfsObject.with_files_stored_remotely + .find_each(batch_size: 10) do |lfs_object| + + lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL) + + logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage") + rescue => e + logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") + end + end end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake deleted file mode 100644 index 100e480bd66..00000000000 --- a/lib/tasks/gitlab/pages.rake +++ /dev/null @@ -1,9 +0,0 @@ -namespace :gitlab do - namespace :pages do - desc 'Ping the pages admin API' - task admin_ping: :gitlab_environment do - Gitlab::PagesClient.ping - puts "OK: gitlab-pages admin API is reachable" - end - end -end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 5d86d6e466c..50774de77c9 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -31,7 +31,6 @@ namespace :gitlab do terminate_all_connections unless Rails.env.production? Rake::Task["db:reset"].invoke - Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError puts "Quitting...".color(:red) diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake deleted file mode 100644 index 5e1ec481ece..00000000000 --- a/lib/tasks/gitlab/traces.rake +++ /dev/null @@ -1,38 +0,0 @@ -require 'logger' -require 'resolv-replace' - -desc "GitLab | Archive legacy traces to trace artifacts" -namespace :gitlab do - namespace :traces do - task archive: :environment do - logger = Logger.new(STDOUT) - logger.info('Archiving legacy traces') - - Ci::Build.finished.without_archived_trace - .order(id: :asc) - .find_in_batches(batch_size: 1000) do |jobs| - job_ids = jobs.map { |job| [job.id] } - - ArchiveTraceWorker.bulk_perform_async(job_ids) - - logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") - end - end - - task migrate: :environment do - logger = Logger.new(STDOUT) - logger.info('Starting transfer of job traces') - - Ci::Build.joins(:project) - .with_archived_trace_stored_locally - .find_each(batch_size: 10) do |build| - - build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE) - - logger.info("Transferred job trace of #{build.id} to object storage") - rescue => e - logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") - end - end - end -end diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 1c93609a006..44536a447c7 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -3,19 +3,7 @@ namespace :gitlab do namespace :migrate do desc "GitLab | Uploads | Migrate all uploaded files to object storage" task all: :environment do - categories = [%w(AvatarUploader Project :avatar), - %w(AvatarUploader Group :avatar), - %w(AvatarUploader User :avatar), - %w(AttachmentUploader Note :attachment), - %w(AttachmentUploader Appearance :logo), - %w(AttachmentUploader Appearance :header_logo), - %w(FaviconUploader Appearance :favicon), - %w(FileUploader Project), - %w(PersonalFileUploader Snippet), - %w(NamespaceFileUploader Snippet), - %w(FileUploader MergeRequest)] - - categories.each do |args| + Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| Rake::Task["gitlab:uploads:migrate"].invoke(*args) Rake::Task["gitlab:uploads:migrate"].reenable end @@ -25,34 +13,23 @@ namespace :gitlab do # The following is the actual rake task that migrates uploads of specified # category to object storage desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' - task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| - batch_size = ENV.fetch('BATCH', 200).to_i - @to_store = ObjectStorage::Store::REMOTE - @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym - @uploader_class = args.uploader_class.constantize - @model_class = args.model_class.constantize - - uploads.each_batch(of: batch_size, &method(:enqueue_batch)) + task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| + Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage end - def enqueue_batch(batch, index) - job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, - @model_class, - @mounted_as, - @to_store) - puts "Enqueued job ##{index}: #{job}" - rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e - # continue for the next batch - puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red) + namespace :migrate_to_local do + desc "GitLab | Uploads | Migrate all uploaded files to local storage" + task all: :environment do + Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args) + Rake::Task["gitlab:uploads:migrate_to_local"].reenable + end + end end - def uploads - Upload.class_eval { include EachBatch } unless Upload < EachBatch - - Upload - .where(store: [nil, ObjectStorage::Store::LOCAL], - uploader: @uploader_class.to_s, - model_type: @model_class.base_class.sti_name) + desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage' + task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| + Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage end end end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index cda88c130bb..4c8f13b63a4 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,14 +1,3 @@ -desc 'GitLab | Sets up PostgreSQL' -task setup_postgresql: :environment do - require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') - require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb') - require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb') - - UsersNameLowerIndex.new.up - ProjectNameLowerIndex.new.up - AddPathIndexToRedirectRoutes.new.up -end - desc 'GitLab | Generate PostgreSQL Password Hash' task :postgresql_md5_hash do require 'digest' diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake deleted file mode 100644 index 4ec4fdd281f..00000000000 --- a/lib/tasks/services.rake +++ /dev/null @@ -1,98 +0,0 @@ -services_template = <<-ERB -# Services - -<% services.each do |service| %> -## <%= service[:title] %> - - -<% unless service[:description].blank? %> -<%= service[:description] %> -<% end %> - - -### Create/Edit <%= service[:title] %> service - -Set <%= service[:title] %> service for a project. -<% unless service[:help].blank? %> - -> <%= service[:help].gsub("\n", ' ') %> - -<% end %> - -``` -PUT /projects/:id/services/<%= service[:dashed_name] %> - -``` - -Parameters: - -<% service[:params].each do |param| %> -- `<%= param[:name] %>` <%= param[:required] ? "(**required**)" : "(optional)" %><%= [" -", param[:description]].join(" ").gsub("\n", '') unless param[:description].blank? %> - -<% end %> - -### Delete <%= service[:title] %> service - -Delete <%= service[:title] %> service for a project. - -``` -DELETE /projects/:id/services/<%= service[:dashed_name] %> - -``` - -### Get <%= service[:title] %> service settings - -Get <%= service[:title] %> service settings for a project. - -``` -GET /projects/:id/services/<%= service[:dashed_name] %> - -``` - -<% end %> -ERB - -namespace :services do - task doc: :environment do - services = Service.available_services_names.map do |s| - service_start = Time.now - klass = "#{s}_service".classify.constantize - - service = klass.new - - service_hash = {} - - service_hash[:title] = service.title - service_hash[:dashed_name] = s.dasherize - service_hash[:description] = service.description - service_hash[:help] = service.help - service_hash[:params] = service.fields.map do |p| - param_hash = {} - - param_hash[:name] = p[:name] - param_hash[:description] = p[:placeholder] || p[:title] - param_hash[:required] = klass.validators_on(p[:name].to_sym).any? do |v| - v.class == ActiveRecord::Validations::PresenceValidator - end - - param_hash - end - service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 } - - puts "Collected data for: #{service.title}, #{Time.now - service_start}" - service_hash - end - - doc_start = Time.now - doc_path = File.join(Rails.root, 'doc', 'api', 'services.md') - - result = ERB.new(services_template, trim_mode: '>') - .result(OpenStruct.new(services: services).instance_eval { binding }) - - File.open(doc_path, 'w') do |f| - f.write result - end - - puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}" - end -end |