Add latest changes from gitlab-org/gitlab@12-4-stable-ee

11 files changed, 73 insertions, 267 deletions

diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake
index 1cac7520227..6e90229830d 100644
--- a/lib/tasks/frontend.rake
+++ b/lib/tasks/frontend.rake
@@ -2,7 +2,10 @@ unless Rails.env.production?
   namespace :frontend do
     desc 'GitLab | Frontend | Generate fixtures for JavaScript tests'
     RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args|
-      args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb')
+      directories = %w[spec]
+      directories << 'ee/spec' if Gitlab.ee?
+      directory_glob = "{#{directories.join(',')}}"
+      args.with_defaults(pattern: "#{directory_glob}/frontend/fixtures/*.rb")
       ENV['NO_KNAPSACK'] = 'true'
       t.pattern = args[:pattern]
       t.rspec_opts = '--format documentation'
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
index 9012e55a70c..0d09fd0a4e3 100644
--- a/lib/tasks/gitlab/artifacts/migrate.rake
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -6,18 +6,31 @@ namespace :gitlab do
   namespace :artifacts do
     task migrate: :environment do
       logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of artifacts')
+      logger.info('Starting transfer of artifacts to remote storage')
 
-      Ci::Build.joins(:project)
-        .with_artifacts_stored_locally
-        .find_each(batch_size: 10) do |build|
+      helper = Gitlab::Artifacts::MigrationHelper.new
 
-        build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
-        build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
+      begin
+        helper.migrate_to_remote_storage do |artifact|
+          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage")
+        end
+      rescue => e
+        logger.error(e.message)
+      end
+    end
+
+    task migrate_to_local: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of artifacts to local storage')
+
+      helper = Gitlab::Artifacts::MigrationHelper.new
 
-        logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage")
+      begin
+        helper.migrate_to_local_storage do |artifact|
+          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage")
+        end
       rescue => e
-        logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+        logger.error(e.message)
       end
     end
   end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 4d854cd178d..0a0ee7b4bfa 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -3,69 +3,6 @@ require 'set'
 
 namespace :gitlab do
   namespace :cleanup do
-    desc "GitLab | Cleanup | Clean namespaces"
-    task dirs: :gitlab_environment do
-      namespaces = Set.new(Namespace.pluck(:path))
-      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
-
-      Gitaly::Server.all.each do |server|
-        all_dirs = Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories(depth: 0)
-          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
-
-        puts "Looking for directories to remove... "
-        all_dirs.each do |dir_path|
-          if remove?
-            begin
-              Gitlab::GitalyClient::NamespaceService.new(server.storage)
-                .remove(dir_path)
-
-              puts "Removed...#{dir_path}"
-            rescue StandardError => e
-              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
-            end
-          else
-            puts "Can be removed: #{dir_path}".color(:red)
-          end
-        end
-      end
-
-      unless remove?
-        puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
-      end
-    end
-
-    desc "GitLab | Cleanup | Clean repositories"
-    task repos: :gitlab_environment do
-      move_suffix = "+orphaned+#{Time.now.to_i}"
-
-      Gitaly::Server.all.each do |server|
-        Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories
-          .each do |path|
-          repo_with_namespace = path.chomp('.git').chomp('.wiki')
-
-          # TODO ignoring hashed repositories for now.  But revisit to fully support
-          # possible orphaned hashed repos
-          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
-          next if Project.find_by_full_path(repo_with_namespace)
-
-          new_path = path + move_suffix
-          puts path.inspect + ' -> ' + new_path.inspect
-
-          begin
-            Gitlab::GitalyClient::NamespaceService
-              .new(server.storage)
-              .rename(path, new_path)
-          rescue StandardError => e
-            puts "Error occurred while moving the repository: #{e.message}".color(:red)
-          end
-        end
-      end
-    end
-
     desc "GitLab | Cleanup | Block users that have been removed in LDAP"
     task block_removed_ldap_users: :gitlab_environment do
       warn_user_is_not_gitlab
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index fd8df015903..902f22684ee 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -11,10 +11,28 @@ namespace :gitlab do
     task compile_docs: :environment do
       renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
 
-      renderer.render
+      renderer.write
 
       puts "Documentation compiled."
     end
+
+    desc 'GitLab | Check if GraphQL docs are up to date'
+    task check_docs: :environment do
+      renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
+
+      doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md'))
+
+      if doc == renderer.contents
+        puts "GraphQL documentation is up to date"
+      else
+        puts '#' * 10
+        puts '#'
+        puts '# GraphQL documentation is outdated! Please update it by running `bundle exec rake gitlab:graphql:compile_docs`.'
+        puts '#'
+        puts '#' * 10
+        abort
+      end
+    end
   end
 end
 
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
index 97c15175a23..4142903d9c3 100644
--- a/lib/tasks/gitlab/lfs/migrate.rake
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -17,5 +17,20 @@ namespace :gitlab do
           logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
         end
     end
+
+    task migrate_to_local: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of LFS files to local storage')
+
+      LfsObject.with_files_stored_remotely
+        .find_each(batch_size: 10) do |lfs_object|
+
+        lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL)
+
+        logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage")
+      rescue => e
+        logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
deleted file mode 100644
index 100e480bd66..00000000000
--- a/lib/tasks/gitlab/pages.rake
+++ /dev/null
@@ -1,9 +0,0 @@
-namespace :gitlab do
-  namespace :pages do
-    desc 'Ping the pages admin API'
-    task admin_ping: :gitlab_environment do
-      Gitlab::PagesClient.ping
-      puts "OK: gitlab-pages admin API is reachable"
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index 5d86d6e466c..50774de77c9 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -31,7 +31,6 @@ namespace :gitlab do
     terminate_all_connections unless Rails.env.production?
 
     Rake::Task["db:reset"].invoke
-    Rake::Task["setup_postgresql"].invoke
     Rake::Task["db:seed_fu"].invoke
   rescue Gitlab::TaskAbortedByUserError
     puts "Quitting...".color(:red)
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
deleted file mode 100644
index 5e1ec481ece..00000000000
--- a/lib/tasks/gitlab/traces.rake
+++ /dev/null
@@ -1,38 +0,0 @@
-require 'logger'
-require 'resolv-replace'
-
-desc "GitLab | Archive legacy traces to trace artifacts"
-namespace :gitlab do
-  namespace :traces do
-    task archive: :environment do
-      logger = Logger.new(STDOUT)
-      logger.info('Archiving legacy traces')
-
-      Ci::Build.finished.without_archived_trace
-        .order(id: :asc)
-        .find_in_batches(batch_size: 1000) do |jobs|
-        job_ids = jobs.map { |job| [job.id] }
-
-        ArchiveTraceWorker.bulk_perform_async(job_ids)
-
-        logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}")
-      end
-    end
-
-    task migrate: :environment do
-      logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of job traces')
-
-      Ci::Build.joins(:project)
-        .with_archived_trace_stored_locally
-        .find_each(batch_size: 10) do |build|
-
-        build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE)
-
-        logger.info("Transferred job trace of #{build.id} to object storage")
-      rescue => e
-        logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
-      end
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
index 1c93609a006..44536a447c7 100644
--- a/lib/tasks/gitlab/uploads/migrate.rake
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -3,19 +3,7 @@ namespace :gitlab do
     namespace :migrate do
       desc "GitLab | Uploads | Migrate all uploaded files to object storage"
       task all: :environment do
-        categories = [%w(AvatarUploader Project :avatar),
-                      %w(AvatarUploader Group :avatar),
-                      %w(AvatarUploader User :avatar),
-                      %w(AttachmentUploader Note :attachment),
-                      %w(AttachmentUploader Appearance :logo),
-                      %w(AttachmentUploader Appearance :header_logo),
-                      %w(FaviconUploader Appearance :favicon),
-                      %w(FileUploader Project),
-                      %w(PersonalFileUploader Snippet),
-                      %w(NamespaceFileUploader Snippet),
-                      %w(FileUploader MergeRequest)]
-
-        categories.each do |args|
+        Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
           Rake::Task["gitlab:uploads:migrate"].invoke(*args)
           Rake::Task["gitlab:uploads:migrate"].reenable
         end
@@ -25,34 +13,23 @@ namespace :gitlab do
     # The following is the actual rake task that migrates uploads of specified
     # category to object storage
     desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage'
-    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
-      batch_size     = ENV.fetch('BATCH', 200).to_i
-      @to_store      = ObjectStorage::Store::REMOTE
-      @mounted_as    = args.mounted_as&.gsub(':', '')&.to_sym
-      @uploader_class = args.uploader_class.constantize
-      @model_class    = args.model_class.constantize
-
-      uploads.each_batch(of: batch_size, &method(:enqueue_batch))
+    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
+      Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage
     end
 
-    def enqueue_batch(batch, index)
-      job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
-                                                         @model_class,
-                                                         @mounted_as,
-                                                         @to_store)
-      puts "Enqueued job ##{index}: #{job}"
-    rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
-      # continue for the next batch
-      puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
+    namespace :migrate_to_local do
+      desc "GitLab | Uploads | Migrate all uploaded files to local storage"
+      task all: :environment do
+        Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
+          Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args)
+          Rake::Task["gitlab:uploads:migrate_to_local"].reenable
+        end
+      end
     end
 
-    def uploads
-      Upload.class_eval { include EachBatch } unless Upload < EachBatch
-
-      Upload
-        .where(store: [nil, ObjectStorage::Store::LOCAL],
-               uploader: @uploader_class.to_s,
-               model_type: @model_class.base_class.sti_name)
+    desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage'
+    task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
+      Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage
     end
   end
 end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index cda88c130bb..4c8f13b63a4 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -1,14 +1,3 @@
-desc 'GitLab | Sets up PostgreSQL'
-task setup_postgresql: :environment do
-  require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb')
-  require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb')
-  require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb')
-
-  UsersNameLowerIndex.new.up
-  ProjectNameLowerIndex.new.up
-  AddPathIndexToRedirectRoutes.new.up
-end
-
 desc 'GitLab | Generate PostgreSQL Password Hash'
 task :postgresql_md5_hash do
   require 'digest'
diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake
deleted file mode 100644
index 4ec4fdd281f..00000000000
--- a/lib/tasks/services.rake
+++ /dev/null
@@ -1,98 +0,0 @@
-services_template = <<-ERB
-# Services
-
-<% services.each do |service| %>
-## <%= service[:title] %>
-
-
-<% unless service[:description].blank? %>
-<%= service[:description] %>
-<% end %>
-
-
-### Create/Edit <%= service[:title] %> service
-
-Set <%= service[:title] %> service for a project.
-<% unless service[:help].blank? %>
-
-> <%= service[:help].gsub("\n", ' ') %>
-
-<% end %>
-
-```
-PUT /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-Parameters:
-
-<% service[:params].each do |param| %>
-- `<%= param[:name] %>` <%= param[:required] ? "(**required**)" : "(optional)"  %><%= [" -",  param[:description]].join(" ").gsub("\n", '') unless param[:description].blank? %>
-
-<% end %>
-
-### Delete <%= service[:title] %> service
-
-Delete <%= service[:title] %> service for a project.
-
-```
-DELETE /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-### Get <%= service[:title] %> service settings
-
-Get <%= service[:title] %> service settings for a project.
-
-```
-GET /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-<% end %>
-ERB
-
-namespace :services do
-  task doc: :environment do
-    services = Service.available_services_names.map do |s|
-      service_start = Time.now
-      klass = "#{s}_service".classify.constantize
-
-      service = klass.new
-
-      service_hash = {}
-
-      service_hash[:title] = service.title
-      service_hash[:dashed_name] = s.dasherize
-      service_hash[:description] = service.description
-      service_hash[:help] = service.help
-      service_hash[:params] = service.fields.map do |p|
-        param_hash = {}
-
-        param_hash[:name] = p[:name]
-        param_hash[:description] = p[:placeholder] || p[:title]
-        param_hash[:required] = klass.validators_on(p[:name].to_sym).any? do |v|
-          v.class == ActiveRecord::Validations::PresenceValidator
-        end
-
-        param_hash
-      end
-      service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 }
-
-      puts "Collected data for: #{service.title}, #{Time.now - service_start}"
-      service_hash
-    end
-
-    doc_start = Time.now
-    doc_path = File.join(Rails.root, 'doc', 'api', 'services.md')
-
-    result = ERB.new(services_template, trim_mode: '>')
-      .result(OpenStruct.new(services: services).instance_eval { binding })
-
-    File.open(doc_path, 'w') do |f|
-      f.write result
-    end
-
-    puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}"
-  end
-end