Add latest changes from gitlab-org/gitlab@13-0-stable-ee

10 files changed, 110 insertions, 62 deletions

diff --git a/lib/tasks/file_hooks.rake b/lib/tasks/file_hooks.rake
index 66d382db612..f767d63fe0d 100644
--- a/lib/tasks/file_hooks.rake
+++ b/lib/tasks/file_hooks.rake
@@ -4,6 +4,11 @@ namespace :file_hooks do
     puts 'Validating file hooks from /file_hooks and /plugins directories'
 
     Gitlab::FileHook.files.each do |file|
+      if File.dirname(file).ends_with?('plugins')
+        puts 'DEPRECATED: /plugins directory is deprecated and will be removed in 14.0. ' \
+          'Please move your files into /file_hooks directory.'
+      end
+
       success, message = Gitlab::FileHook.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA)
 
       if success
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index 8cf7c9e89f0..3833689e07e 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -7,7 +7,7 @@ namespace :gemojione do
     aliases = {}
 
     index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json')
-    index = JSON.parse(File.read(index_file))
+    index = Gitlab::Json.parse(File.read(index_file))
 
     index.each_pair do |key, data|
       data['aliases'].each do |a|
@@ -19,7 +19,7 @@ namespace :gemojione do
 
     out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
     File.open(out, 'w') do |handle|
-      handle.write(JSON.pretty_generate(aliases, indent: '   ', space: '', space_before: ''))
+      handle.write(Gitlab::Json.pretty_generate(aliases, indent: '   ', space: '', space_before: ''))
     end
   end
 
@@ -58,7 +58,7 @@ namespace :gemojione do
 
     out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
     File.open(out, 'w') do |handle|
-      handle.write(JSON.pretty_generate(resultant_emoji_map))
+      handle.write(Gitlab::Json.pretty_generate(resultant_emoji_map))
     end
   end
 
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index ee47f71af93..fc55d9704d1 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -13,10 +13,9 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
 
       version = Gitlab::GitalyClient.expected_server_version
 
-      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
-
-      command = %w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE]
+      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
 
+      command = []
       _, status = Gitlab::Popen.popen(%w[which gmake])
       command << (status.zero? ? 'gmake' : 'make')
 
@@ -31,7 +30,7 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       Dir.chdir(args.dir) do
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
-          Bundler.with_original_env { run_command!(command) }
+          Bundler.with_original_env { Gitlab::Popen.popen(command, nil, { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }) }
         end
       end
     end
diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake
index c9c212fbe4d..4bdc62c9319 100644
--- a/lib/tasks/gitlab/import_export/export.rake
+++ b/lib/tasks/gitlab/import_export/export.rake
@@ -3,12 +3,12 @@
 # Export project to archive
 #
 # @example
-#   bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]"
+#   bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz]"
 #
 namespace :gitlab do
   namespace :import_export do
     desc 'GitLab | Import/Export | EXPERIMENTAL | Export large project archives'
-    task :export, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
+    task :export, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
       # Load it here to avoid polluting Rake tasks with Sidekiq test warnings
       require 'sidekiq/testing'
 
@@ -18,6 +18,7 @@ namespace :gitlab do
         warn_user_is_not_gitlab
 
         if ENV['EXPORT_DEBUG'].present?
+          Gitlab::Utils::Measuring.logger = logger
           ActiveRecord::Base.logger = logger
           logger.level = Logger::DEBUG
         else
@@ -29,7 +30,6 @@ namespace :gitlab do
           project_path:   args.project_path,
           username:       args.username,
           file_path:      args.archive_path,
-          measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled),
           logger: logger
         )
 
diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake
index 7e2162a7774..2702b530334 100644
--- a/lib/tasks/gitlab/import_export/import.rake
+++ b/lib/tasks/gitlab/import_export/import.rake
@@ -7,12 +7,12 @@
 #   2. Performs Sidekiq job synchronously
 #
 # @example
-#   bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz, true]"
+#   bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz]"
 #
 namespace :gitlab do
   namespace :import_export do
     desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives'
-    task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
+    task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
       # Load it here to avoid polluting Rake tasks with Sidekiq test warnings
       require 'sidekiq/testing'
 
@@ -22,6 +22,7 @@ namespace :gitlab do
         warn_user_is_not_gitlab
 
         if ENV['IMPORT_DEBUG'].present?
+          Gitlab::Utils::Measuring.logger = logger
           ActiveRecord::Base.logger = logger
           logger.level = Logger::DEBUG
         else
@@ -33,7 +34,6 @@ namespace :gitlab do
           project_path:   args.project_path,
           username:       args.username,
           file_path:      args.archive_path,
-          measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled),
           logger: logger
         )
 
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 6586699f8ba..d6e62a5c550 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -12,7 +12,7 @@ namespace :gitlab do
       gitlab_url += '/' unless gitlab_url.end_with?('/')
       target_dir = Gitlab.config.gitlab_shell.path
 
-      checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir)
+      checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir, clone_opts: %w[--depth 1])
 
       # Make sure we're on the right tag
       Dir.chdir(target_dir) do
diff --git a/lib/tasks/gitlab/snippets.rake b/lib/tasks/gitlab/snippets.rake
new file mode 100644
index 00000000000..c391cecfdbc
--- /dev/null
+++ b/lib/tasks/gitlab/snippets.rake
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :snippets do
+    DEFAULT_LIMIT = 100
+
+    # @example
+    #   bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4
+    #   bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 LIMIT=50
+    desc 'GitLab | Migrate specific snippets to git'
+    task :migrate, [:ids] => :environment do |_, args|
+      unless ENV['SNIPPET_IDS'].presence
+        raise "Please supply the list of ids through the SNIPPET_IDS env var"
+      end
+
+      raise "Invalid limit value" if limit.zero?
+
+      if migration_running?
+        raise "There are already snippet migrations running. Please wait until they are finished."
+      end
+
+      ids = parse_snippet_ids!
+
+      puts "Starting the migration..."
+      Gitlab::BackgroundMigration::BackfillSnippetRepositories.new.perform_by_ids(ids)
+
+      list_non_migrated = non_migrated_snippets.where(id: ids)
+
+      if list_non_migrated.exists?
+        puts "The following snippets couldn't be migrated:"
+        puts list_non_migrated.pluck(:id).join(',')
+      else
+        puts "All snippets were migrated successfully"
+      end
+    end
+
+    def parse_snippet_ids!
+      ids = ENV['SNIPPET_IDS'].delete(' ').split(',').map do |id|
+        id.to_i.tap do |value|
+          raise "Invalid id provided" if value.zero?
+        end
+      end
+
+      if ids.size > limit
+        raise "The number of ids provided is higher than #{limit}. You can update this limit by using the env var `LIMIT`"
+      end
+
+      ids
+    end
+
+    # @example
+    #   bundle exec rake gitlab:snippets:migration_status
+    desc 'GitLab | Show whether there are snippet background migrations running'
+    task migration_status: :environment do
+      if migration_running?
+        puts "There are snippet migrations running"
+      else
+        puts "There are no snippet migrations running"
+      end
+    end
+
+    def migration_running?
+      Sidekiq::ScheduledSet.new.any? { |r| r.klass == 'BackgroundMigrationWorker' && r.args[0] == 'BackfillSnippetRepositories' }
+    end
+
+    # @example
+    #   bundle exec rake gitlab:snippets:list_non_migrated
+    #   bundle exec rake gitlab:snippets:list_non_migrated LIMIT=50
+    desc 'GitLab | Show non migrated snippets'
+    task list_non_migrated: :environment do
+      raise "Invalid limit value" if limit.zero?
+
+      non_migrated_count = non_migrated_snippets.count
+      if non_migrated_count.zero?
+        puts "All snippets have been successfully migrated"
+      else
+        puts "There are #{non_migrated_count} snippets that haven't been migrated. Showing a batch of ids of those snippets:\n"
+        puts non_migrated_snippets.limit(limit).pluck(:id).join(',')
+      end
+    end
+
+    def non_migrated_snippets
+      @non_migrated_snippets ||= Snippet.select(:id).where.not(id: SnippetRepository.select(:snippet_id))
+    end
+
+    # There are problems with the specs if we memoize this value
+    def limit
+      ENV['LIMIT'] ? ENV['LIMIT'].to_i : DEFAULT_LIMIT
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake
deleted file mode 100644
index 6f101aea303..00000000000
--- a/lib/tasks/gitlab/track_deployment.rake
+++ /dev/null
@@ -1,9 +0,0 @@
-namespace :gitlab do
-  desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring'
-  task track_deployment: :environment do
-    metric = Gitlab::Metrics::Metric
-      .new('deployments', version: Gitlab::VERSION)
-
-    Gitlab::Metrics.submit_metrics([metric.to_hash])
-  end
-end
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index bae3e4e8001..53343c8f8ff 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -12,7 +12,7 @@ namespace :gitlab do
 
       version = Gitlab::Workhorse.version
 
-      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
+      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
 
       _, status = Gitlab::Popen.popen(%w[which gmake])
       command = status.zero? ? 'gmake' : 'make'
diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake
deleted file mode 100644
index d74878835fd..00000000000
--- a/lib/tasks/sidekiq.rake
+++ /dev/null
@@ -1,38 +0,0 @@
-namespace :sidekiq do
-  def deprecation_warning!
-    warn <<~WARNING
-      This task is deprecated and will be removed in 13.0 as it is thought to be unused.
-
-      If you are using this task, please comment on the below issue:
-        https://gitlab.com/gitlab-org/gitlab/issues/196731
-    WARNING
-  end
-
-  desc '[DEPRECATED] GitLab | Sidekiq | Stop sidekiq'
-  task :stop do
-    deprecation_warning!
-
-    system(*%w(bin/background_jobs stop))
-  end
-
-  desc '[DEPRECATED] GitLab | Sidekiq | Start sidekiq'
-  task :start do
-    deprecation_warning!
-
-    system(*%w(bin/background_jobs start))
-  end
-
-  desc '[DEPRECATED] GitLab | Sidekiq | Restart sidekiq'
-  task :restart do
-    deprecation_warning!
-
-    system(*%w(bin/background_jobs restart))
-  end
-
-  desc '[DEPRECATED] GitLab | Sidekiq | Start sidekiq with launchd on Mac OS X'
-  task :launchd do
-    deprecation_warning!
-
-    system(*%w(bin/background_jobs start_silent))
-  end
-end