Add latest changes from gitlab-org/gitlab@master

2 files changed, 54 insertions, 215 deletions

diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake
index 6cedf4e8cf1..ae54636fdd3 100644
--- a/lib/tasks/gitlab/import_export/export.rake
+++ b/lib/tasks/gitlab/import_export/export.rake
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'gitlab/with_request_store'
-
 # Export project to archive
 #
 # @example
@@ -14,81 +12,36 @@ namespace :gitlab do
       # Load it here to avoid polluting Rake tasks with Sidekiq test warnings
       require 'sidekiq/testing'
 
-      warn_user_is_not_gitlab
-
-      if ENV['IMPORT_DEBUG'].present?
-        ActiveRecord::Base.logger = Logger.new(STDOUT)
-        Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
-      end
-
-      GitlabProjectExport.new(
-        namespace_path: args.namespace_path,
-        project_path:   args.project_path,
-        username:       args.username,
-        file_path:      args.archive_path,
-        measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled)
-      ).export
-    end
-  end
-end
-
-class GitlabProjectExport
-  include Gitlab::WithRequestStore
-
-  def initialize(opts)
-    @project_path = opts.fetch(:project_path)
-    @file_path = opts.fetch(:file_path)
-    @current_user = User.find_by_username(opts.fetch(:username))
-    namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path))
-    @project = namespace.projects.find_by_path(@project_path)
-    @measurement_enabled = opts.fetch(:measurement_enabled)
-    @measurable = Gitlab::Utils::Measuring.new if @measurement_enabled
-  end
-
-  def export
-    validate_project
-    validate_file_path
-
-    with_export do
-      ::Projects::ImportExport::ExportService.new(project, current_user)
-        .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path))
-    end
-
-    puts 'Done!'
-  rescue StandardError => e
-    puts "Exception: #{e.message}"
-    puts e.backtrace
-    exit 1
-  end
-
-  private
-
-  attr_reader :measurable, :project, :current_user, :file_path, :project_path
-
-  def validate_project
-    unless project
-      puts "Error: Project with path: #{project_path} was not found. Please provide correct project path"
-      exit 1
-    end
-  end
-
-  def validate_file_path
-    directory = File.dirname(file_path)
-    unless Dir.exist?(directory)
-      puts "Error: Invalid file path: #{file_path}. Please provide correct file path"
-      exit 1
-    end
-  end
-
-  def with_export
-    with_request_store do
-      ::Gitlab::GitalyClient.allow_n_plus_1_calls do
-        measurement_enabled? ? measurable.with_measuring { yield } : yield
+      logger = Logger.new($stdout)
+
+      begin
+        warn_user_is_not_gitlab
+
+        if ENV['EXPORT_DEBUG'].present?
+          ActiveRecord::Base.logger = logger
+          Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
+          logger.level = Logger::DEBUG
+        else
+          logger.level = Logger::INFO
+        end
+
+        task = Gitlab::ImportExport::Project::ExportTask.new(
+          namespace_path: args.namespace_path,
+          project_path:   args.project_path,
+          username:       args.username,
+          file_path:      args.archive_path,
+          measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled),
+          logger: logger
+        )
+
+        success = task.export
+
+        exit(success)
+      rescue StandardError => e
+        logger.error "Exception: #{e.message}"
+        logger.debug e.backtrace
+        exit 1
       end
     end
   end
-
-  def measurement_enabled?
-    @measurement_enabled
-  end
 end
diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake
index 4ed724a5c82..6e2d0e75da8 100644
--- a/lib/tasks/gitlab/import_export/import.rake
+++ b/lib/tasks/gitlab/import_export/import.rake
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'gitlab/with_request_store'
-
 # Import large project archives
 #
 # This task:
@@ -18,148 +16,36 @@ namespace :gitlab do
       # Load it here to avoid polluting Rake tasks with Sidekiq test warnings
       require 'sidekiq/testing'
 
-      warn_user_is_not_gitlab
-
-      if ENV['IMPORT_DEBUG'].present?
-        ActiveRecord::Base.logger = Logger.new(STDOUT)
-      end
-
-      GitlabProjectImport.new(
-        namespace_path: args.namespace_path,
-        project_path:   args.project_path,
-        username:       args.username,
-        file_path:      args.archive_path,
-        measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled)
-      ).import
-    end
-  end
-end
-
-class GitlabProjectImport
-  include Gitlab::WithRequestStore
-
-  def initialize(opts)
-    @project_path = opts.fetch(:project_path)
-    @file_path    = opts.fetch(:file_path)
-    @namespace    = Namespace.find_by_full_path(opts.fetch(:namespace_path))
-    @current_user = User.find_by_username(opts.fetch(:username))
-    @measurement_enabled = opts.fetch(:measurement_enabled)
-    @measurement = Gitlab::Utils::Measuring.new if @measurement_enabled
-  end
-
-  def import
-    show_import_start_message
+      logger = Logger.new($stdout)
 
-    run_isolated_sidekiq_job
+      begin
+        warn_user_is_not_gitlab
 
-    show_import_failures_count
-
-    if project&.import_state&.last_error
-      puts "ERROR: #{project.import_state.last_error}"
-      exit 1
-    elsif project.errors.any?
-      puts "ERROR: #{project.errors.full_messages.join(', ')}"
-      exit 1
-    else
-      puts 'Done!'
-    end
-  rescue StandardError => e
-    puts "Exception: #{e.message}"
-    puts e.backtrace
-    exit 1
-  end
-
-  private
-
-  attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path
-
-  def measurement_enabled?
-    @measurement_enabled
-  end
-
-  # We want to ensure that all Sidekiq jobs are executed
-  # synchronously as part of that process.
-  # This ensures that all expensive operations do not escape
-  # to general Sidekiq clusters/nodes.
-  def with_isolated_sidekiq_job
-    Sidekiq::Testing.fake! do
-      with_request_store do
-        # If you are attempting to import a large project into a development environment,
-        # you may see Gitaly throw an error about too many calls or invocations.
-        # This is due to a n+1 calls limit being set for development setups (not enforced in production)
-        # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
-        # For development setups, this code-path will be excluded from n+1 detection.
-        ::Gitlab::GitalyClient.allow_n_plus_1_calls do
-          measurement_enabled? ? measurement.with_measuring { yield } : yield
+        if ENV['IMPORT_DEBUG'].present?
+          ActiveRecord::Base.logger = logger
+          Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
+          logger.level = Logger::DEBUG
+        else
+          logger.level = Logger::INFO
         end
-      end
-
-      true
-    end
-  end
-
-  def run_isolated_sidekiq_job
-    with_isolated_sidekiq_job do
-      @project = create_project
-
-      execute_sidekiq_job
-    end
-  end
-
-  def create_project
-    # We are disabling ObjectStorage for `import`
-    # as it is too slow to handle big archives:
-    # 1. DB transaction timeouts on upload
-    # 2. Download of archive before unpacking
-    disable_upload_object_storage do
-      service = Projects::GitlabProjectsImportService.new(
-        current_user,
-        {
-          namespace_id: namespace.id,
-          path:         project_path,
-          file:         File.open(file_path)
-        }
-      )
-
-      service.execute
-    end
-  end
-
-  def execute_sidekiq_job
-    Sidekiq::Worker.drain_all
-  end
 
-  def disable_upload_object_storage
-    overwrite_uploads_setting('background_upload', false) do
-      overwrite_uploads_setting('direct_upload', false) do
-        yield
+        task = Gitlab::ImportExport::Project::ImportTask.new(
+          namespace_path: args.namespace_path,
+          project_path:   args.project_path,
+          username:       args.username,
+          file_path:      args.archive_path,
+          measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled),
+          logger: logger
+        )
+
+        success = task.import
+
+        exit(success)
+      rescue StandardError => e
+        logger.error "Exception: #{e.message}"
+        logger.debug e.backtrace
+        exit 1
       end
     end
   end
-
-  def overwrite_uploads_setting(key, value)
-    old_value = Settings.uploads.object_store[key]
-    Settings.uploads.object_store[key] = value
-
-    yield
-
-  ensure
-    Settings.uploads.object_store[key] = old_value
-  end
-
-  def full_path
-    "#{namespace.full_path}/#{project_path}"
-  end
-
-  def show_import_start_message
-    puts "Importing GitLab export: #{file_path} into GitLab" \
-      " #{full_path}" \
-      " as #{current_user.name}"
-  end
-
-  def show_import_failures_count
-    return unless project.import_failures.exists?
-
-    puts "Total number of not imported relations: #{project.import_failures.count}"
-  end
 end