diff options
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/gitlab/import_export/export.rake | 94 | ||||
-rw-r--r-- | lib/tasks/gitlab/import_export/import.rake | 89 |
2 files changed, 116 insertions, 67 deletions
diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake new file mode 100644 index 00000000000..6cedf4e8cf1 --- /dev/null +++ b/lib/tasks/gitlab/import_export/export.rake @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'gitlab/with_request_store' + +# Export project to archive +# +# @example +# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]" +# +namespace :gitlab do + namespace :import_export do + desc 'GitLab | Import/Export | EXPERIMENTAL | Export large project archives' + task :export, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args| + # Load it here to avoid polluting Rake tasks with Sidekiq test warnings + require 'sidekiq/testing' + + warn_user_is_not_gitlab + + if ENV['IMPORT_DEBUG'].present? + ActiveRecord::Base.logger = Logger.new(STDOUT) + Gitlab::Metrics::Exporter::SidekiqExporter.instance.start + end + + GitlabProjectExport.new( + namespace_path: args.namespace_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled) + ).export + end + end +end + +class GitlabProjectExport + include Gitlab::WithRequestStore + + def initialize(opts) + @project_path = opts.fetch(:project_path) + @file_path = opts.fetch(:file_path) + @current_user = User.find_by_username(opts.fetch(:username)) + namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) + @project = namespace.projects.find_by_path(@project_path) + @measurement_enabled = opts.fetch(:measurement_enabled) + @measurable = Gitlab::Utils::Measuring.new if @measurement_enabled + end + + def export + validate_project + validate_file_path + + with_export do + ::Projects::ImportExport::ExportService.new(project, current_user) + .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path)) + end + + puts 'Done!' + rescue StandardError => e + puts "Exception: #{e.message}" + puts e.backtrace + exit 1 + end + + private + + attr_reader :measurable, :project, :current_user, :file_path, :project_path + + def validate_project + unless project + puts "Error: Project with path: #{project_path} was not found. Please provide correct project path" + exit 1 + end + end + + def validate_file_path + directory = File.dirname(file_path) + unless Dir.exist?(directory) + puts "Error: Invalid file path: #{file_path}. Please provide correct file path" + exit 1 + end + end + + def with_export + with_request_store do + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + measurement_enabled? ? measurable.with_measuring { yield } : yield + end + end + end + + def measurement_enabled? + @measurement_enabled + end +end diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index c832cba0287..4ed724a5c82 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'gitlab/with_request_store' + # Import large project archives # # This task: @@ -27,19 +29,22 @@ namespace :gitlab do project_path: args.project_path, username: args.username, file_path: args.archive_path, - measurement_enabled: args.measurement_enabled == 'true' + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled) ).import end end end class GitlabProjectImport + include Gitlab::WithRequestStore + def initialize(opts) @project_path = opts.fetch(:project_path) @file_path = opts.fetch(:file_path) @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) @current_user = User.find_by_username(opts.fetch(:username)) @measurement_enabled = opts.fetch(:measurement_enabled) + @measurement = Gitlab::Utils::Measuring.new if @measurement_enabled end def import @@ -49,11 +54,11 @@ class GitlabProjectImport show_import_failures_count - if @project&.import_state&.last_error - puts "ERROR: #{@project.import_state.last_error}" + if project&.import_state&.last_error + puts "ERROR: #{project.import_state.last_error}" exit 1 - elsif @project.errors.any? - puts "ERROR: #{@project.errors.full_messages.join(', ')}" + elsif project.errors.any? + puts "ERROR: #{project.errors.full_messages.join(', ')}" exit 1 else puts 'Done!' @@ -66,60 +71,10 @@ class GitlabProjectImport private - def with_request_store - RequestStore.begin! - yield - ensure - RequestStore.end! - RequestStore.clear! - end - - def with_count_queries(&block) - count = 0 - - counter_f = ->(name, started, finished, unique_id, payload) { - unless payload[:name].in? %w[CACHE SCHEMA] - count += 1 - end - } - - ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block) - - puts "Number of sql calls: #{count}" - end - - def with_gc_counter - gc_counts_before = GC.stat.select { |k, v| k =~ /count/ } - yield - gc_counts_after = GC.stat.select { |k, v| k =~ /count/ } - stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb } - puts "Total GC count: #{stats[:count]}" - puts "Minor GC count: #{stats[:minor_gc_count]}" - puts "Major GC count: #{stats[:major_gc_count]}" - end - - def with_measure_time - timing = Benchmark.realtime do - yield - end - - time = Time.at(timing).utc.strftime("%H:%M:%S") - puts "Time to finish: #{time}" - end - - def with_measuring - puts "Measuring enabled..." - with_gc_counter do - with_count_queries do - with_measure_time do - yield - end - end - end - end + attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path def measurement_enabled? - @measurement_enabled != false + @measurement_enabled end # We want to ensure that all Sidekiq jobs are executed @@ -135,7 +90,7 @@ class GitlabProjectImport # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 # For development setups, this code-path will be excluded from n+1 detection. ::Gitlab::GitalyClient.allow_n_plus_1_calls do - measurement_enabled? ? with_measuring { yield } : yield + measurement_enabled? ? measurement.with_measuring { yield } : yield end end @@ -158,11 +113,11 @@ class GitlabProjectImport # 2. Download of archive before unpacking disable_upload_object_storage do service = Projects::GitlabProjectsImportService.new( - @current_user, + current_user, { - namespace_id: @namespace.id, - path: @project_path, - file: File.open(@file_path) + namespace_id: namespace.id, + path: project_path, + file: File.open(file_path) } ) @@ -193,18 +148,18 @@ class GitlabProjectImport end def full_path - "#{@namespace.full_path}/#{@project_path}" + "#{namespace.full_path}/#{project_path}" end def show_import_start_message - puts "Importing GitLab export: #{@file_path} into GitLab" \ + puts "Importing GitLab export: #{file_path} into GitLab" \ " #{full_path}" \ - " as #{@current_user.name}" + " as #{current_user.name}" end def show_import_failures_count - return unless @project.import_failures.exists? + return unless project.import_failures.exists? - puts "Total number of not imported relations: #{@project.import_failures.count}" + puts "Total number of not imported relations: #{project.import_failures.count}" end end |