diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/gitlab/git/blob.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb | 19 | ||||
| -rw-r--r-- | lib/gitlab/profiler.rb | 32 | ||||
| -rw-r--r-- | lib/gitlab/sidekiq_config/cli_methods.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/sidekiq_middleware/request_store_middleware.rb | 10 | ||||
| -rw-r--r-- | lib/gitlab/utils/measuring.rb | 74 | ||||
| -rw-r--r-- | lib/gitlab/with_request_store.rb | 13 | ||||
| -rw-r--r-- | lib/tasks/gitlab/import_export/export.rake | 94 | ||||
| -rw-r--r-- | lib/tasks/gitlab/import_export/import.rake | 89 |
9 files changed, 247 insertions, 90 deletions
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index caa1314dd7f..5579449bf57 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -165,7 +165,9 @@ module Gitlab end def truncated? - size && (size > loaded_size) + return false unless size && loaded_size + + size > loaded_size end # Valid LFS object pointer is a text file consisting of diff --git a/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb new file mode 100644 index 00000000000..2e3136936f8 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module AfterExportStrategies + class MoveFileStrategy < BaseAfterExportStrategy + def initialize(archive_path:) + @archive_path = archive_path + end + + private + + def strategy_execute + FileUtils.mv(project.export_file.path, @archive_path) + end + end + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index f47ccb8fed9..e10cdf0d8fb 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -2,6 +2,8 @@ module Gitlab module Profiler + extend WithRequestStore + FILTERED_STRING = '[FILTERED]' IGNORE_BACKTRACES = %w[ @@ -58,28 +60,26 @@ module Gitlab logger = create_custom_logger(logger, private_token: private_token) - RequestStore.begin! - - # Make an initial call for an asset path in development mode to avoid - # sprockets dominating the profiler output. - ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development? + result = with_request_store do + # Make an initial call for an asset path in development mode to avoid + # sprockets dominating the profiler output. + ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development? - # Rails loads internationalization files lazily the first time a - # translation is needed. Running this prevents this overhead from showing - # up in profiles. - ::I18n.t('.')[:test_string] + # Rails loads internationalization files lazily the first time a + # translation is needed. Running this prevents this overhead from showing + # up in profiles. + ::I18n.t('.')[:test_string] - # Remove API route mounting from the profile. - app.get('/api/v4/users') + # Remove API route mounting from the profile. + app.get('/api/v4/users') - result = with_custom_logger(logger) do - with_user(user) do - RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend + with_custom_logger(logger) do + with_user(user) do + RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend + end end end - RequestStore.end! - log_load_times_by_model(logger) result diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index 8f19b557d24..c95ba6faf1e 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -21,7 +21,7 @@ module Gitlab QUERY_OR_OPERATOR = '|' QUERY_AND_OPERATOR = '&' QUERY_CONCATENATE_OPERATOR = ',' - QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze + QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze QUERY_PREDICATES = { feature_category: :to_sym, diff --git a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb index 8824f81e8e3..f6142bd6ca5 100644 --- a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb +++ b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb @@ -3,12 +3,12 @@ module Gitlab module SidekiqMiddleware class RequestStoreMiddleware + include Gitlab::WithRequestStore + def call(worker, job, queue) - RequestStore.begin! - yield - ensure - RequestStore.end! - RequestStore.clear! + with_request_store do + yield + end end end end diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb new file mode 100644 index 00000000000..20c57e777d8 --- /dev/null +++ b/lib/gitlab/utils/measuring.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'prometheus/pid_provider' + +module Gitlab + module Utils + class Measuring + def initialize(logger: Logger.new($stdout)) + @logger = logger + end + + def with_measuring + logger.info "Measuring enabled..." + with_gc_counter do + with_count_queries do + with_measure_time do + yield + end + end + end + + logger.info "Memory usage: #{Gitlab::Metrics::System.memory_usage.to_f / 1024 / 1024} MiB" + logger.info "Label: #{::Prometheus::PidProvider.worker_id}" + end + + private + + attr_reader :logger + + def with_count_queries(&block) + count = 0 + + counter_f = ->(_name, _started, _finished, _unique_id, payload) { + count += 1 unless payload[:name].in? %w[CACHE SCHEMA] + } + + ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block) + + logger.info "Number of sql calls: #{count}" + end + + def with_gc_counter + gc_counts_before = GC.stat.select { |k, _v| k =~ /count/ } + yield + gc_counts_after = GC.stat.select { |k, _v| k =~ /count/ } + stats = gc_counts_before.merge(gc_counts_after) { |_k, vb, va| va - vb } + + logger.info "Total GC count: #{stats[:count]}" + logger.info "Minor GC count: #{stats[:minor_gc_count]}" + logger.info "Major GC count: #{stats[:major_gc_count]}" + end + + def with_measure_time + timing = Benchmark.realtime do + yield + end + + logger.info "Time to finish: #{duration_in_numbers(timing)}" + end + + def duration_in_numbers(duration_in_seconds) + seconds = duration_in_seconds % 1.minute + minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute) + hours = duration_in_seconds / 1.hour + + if hours == 0 + "%02d:%02d" % [minutes, seconds] + else + "%02d:%02d:%02d" % [hours, minutes, seconds] + end + end + end + end +end diff --git a/lib/gitlab/with_request_store.rb b/lib/gitlab/with_request_store.rb new file mode 100644 index 00000000000..d6c05e1e256 --- /dev/null +++ b/lib/gitlab/with_request_store.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module WithRequestStore + def with_request_store + RequestStore.begin! + yield + ensure + RequestStore.end! + RequestStore.clear! + end + end +end diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake new file mode 100644 index 00000000000..6cedf4e8cf1 --- /dev/null +++ b/lib/tasks/gitlab/import_export/export.rake @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'gitlab/with_request_store' + +# Export project to archive +# +# @example +# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]" +# +namespace :gitlab do + namespace :import_export do + desc 'GitLab | Import/Export | EXPERIMENTAL | Export large project archives' + task :export, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args| + # Load it here to avoid polluting Rake tasks with Sidekiq test warnings + require 'sidekiq/testing' + + warn_user_is_not_gitlab + + if ENV['IMPORT_DEBUG'].present? + ActiveRecord::Base.logger = Logger.new(STDOUT) + Gitlab::Metrics::Exporter::SidekiqExporter.instance.start + end + + GitlabProjectExport.new( + namespace_path: args.namespace_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled) + ).export + end + end +end + +class GitlabProjectExport + include Gitlab::WithRequestStore + + def initialize(opts) + @project_path = opts.fetch(:project_path) + @file_path = opts.fetch(:file_path) + @current_user = User.find_by_username(opts.fetch(:username)) + namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) + @project = namespace.projects.find_by_path(@project_path) + @measurement_enabled = opts.fetch(:measurement_enabled) + @measurable = Gitlab::Utils::Measuring.new if @measurement_enabled + end + + def export + validate_project + validate_file_path + + with_export do + ::Projects::ImportExport::ExportService.new(project, current_user) + .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path)) + end + + puts 'Done!' + rescue StandardError => e + puts "Exception: #{e.message}" + puts e.backtrace + exit 1 + end + + private + + attr_reader :measurable, :project, :current_user, :file_path, :project_path + + def validate_project + unless project + puts "Error: Project with path: #{project_path} was not found. Please provide correct project path" + exit 1 + end + end + + def validate_file_path + directory = File.dirname(file_path) + unless Dir.exist?(directory) + puts "Error: Invalid file path: #{file_path}. Please provide correct file path" + exit 1 + end + end + + def with_export + with_request_store do + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + measurement_enabled? ? measurable.with_measuring { yield } : yield + end + end + end + + def measurement_enabled? + @measurement_enabled + end +end diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index c832cba0287..4ed724a5c82 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'gitlab/with_request_store' + # Import large project archives # # This task: @@ -27,19 +29,22 @@ namespace :gitlab do project_path: args.project_path, username: args.username, file_path: args.archive_path, - measurement_enabled: args.measurement_enabled == 'true' + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled) ).import end end end class GitlabProjectImport + include Gitlab::WithRequestStore + def initialize(opts) @project_path = opts.fetch(:project_path) @file_path = opts.fetch(:file_path) @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) @current_user = User.find_by_username(opts.fetch(:username)) @measurement_enabled = opts.fetch(:measurement_enabled) + @measurement = Gitlab::Utils::Measuring.new if @measurement_enabled end def import @@ -49,11 +54,11 @@ class GitlabProjectImport show_import_failures_count - if @project&.import_state&.last_error - puts "ERROR: #{@project.import_state.last_error}" + if project&.import_state&.last_error + puts "ERROR: #{project.import_state.last_error}" exit 1 - elsif @project.errors.any? - puts "ERROR: #{@project.errors.full_messages.join(', ')}" + elsif project.errors.any? + puts "ERROR: #{project.errors.full_messages.join(', ')}" exit 1 else puts 'Done!' @@ -66,60 +71,10 @@ class GitlabProjectImport private - def with_request_store - RequestStore.begin! - yield - ensure - RequestStore.end! - RequestStore.clear! - end - - def with_count_queries(&block) - count = 0 - - counter_f = ->(name, started, finished, unique_id, payload) { - unless payload[:name].in? %w[CACHE SCHEMA] - count += 1 - end - } - - ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block) - - puts "Number of sql calls: #{count}" - end - - def with_gc_counter - gc_counts_before = GC.stat.select { |k, v| k =~ /count/ } - yield - gc_counts_after = GC.stat.select { |k, v| k =~ /count/ } - stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb } - puts "Total GC count: #{stats[:count]}" - puts "Minor GC count: #{stats[:minor_gc_count]}" - puts "Major GC count: #{stats[:major_gc_count]}" - end - - def with_measure_time - timing = Benchmark.realtime do - yield - end - - time = Time.at(timing).utc.strftime("%H:%M:%S") - puts "Time to finish: #{time}" - end - - def with_measuring - puts "Measuring enabled..." - with_gc_counter do - with_count_queries do - with_measure_time do - yield - end - end - end - end + attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path def measurement_enabled? - @measurement_enabled != false + @measurement_enabled end # We want to ensure that all Sidekiq jobs are executed @@ -135,7 +90,7 @@ class GitlabProjectImport # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 # For development setups, this code-path will be excluded from n+1 detection. ::Gitlab::GitalyClient.allow_n_plus_1_calls do - measurement_enabled? ? with_measuring { yield } : yield + measurement_enabled? ? measurement.with_measuring { yield } : yield end end @@ -158,11 +113,11 @@ class GitlabProjectImport # 2. Download of archive before unpacking disable_upload_object_storage do service = Projects::GitlabProjectsImportService.new( - @current_user, + current_user, { - namespace_id: @namespace.id, - path: @project_path, - file: File.open(@file_path) + namespace_id: namespace.id, + path: project_path, + file: File.open(file_path) } ) @@ -193,18 +148,18 @@ class GitlabProjectImport end def full_path - "#{@namespace.full_path}/#{@project_path}" + "#{namespace.full_path}/#{project_path}" end def show_import_start_message - puts "Importing GitLab export: #{@file_path} into GitLab" \ + puts "Importing GitLab export: #{file_path} into GitLab" \ " #{full_path}" \ - " as #{@current_user.name}" + " as #{current_user.name}" end def show_import_failures_count - return unless @project.import_failures.exists? + return unless project.import_failures.exists? - puts "Total number of not imported relations: #{@project.import_failures.count}" + puts "Total number of not imported relations: #{project.import_failures.count}" end end |