diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/gitlab/background_migration/link_lfs_objects.rb | 58 | ||||
| -rw-r--r-- | lib/gitlab/cycle_analytics/usage_data.rb | 21 | ||||
| -rw-r--r-- | lib/gitlab/import_export/error.rb | 9 | ||||
| -rw-r--r-- | lib/gitlab/import_export/group/tree_restorer.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/import_export/project/base_task.rb | 41 | ||||
| -rw-r--r-- | lib/gitlab/import_export/project/export_task.rb | 43 | ||||
| -rw-r--r-- | lib/gitlab/import_export/project/import_task.rb | 110 | ||||
| -rw-r--r-- | lib/gitlab/import_export/shared.rb | 8 | ||||
| -rw-r--r-- | lib/gitlab/usage_data.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/utils/measuring.rb | 5 | ||||
| -rw-r--r-- | lib/tasks/gitlab/import_export/export.rake | 105 | ||||
| -rw-r--r-- | lib/tasks/gitlab/import_export/import.rake | 164 |
12 files changed, 284 insertions, 292 deletions
diff --git a/lib/gitlab/background_migration/link_lfs_objects.rb b/lib/gitlab/background_migration/link_lfs_objects.rb index 3131b5d5125..69c03f617bf 100644 --- a/lib/gitlab/background_migration/link_lfs_objects.rb +++ b/lib/gitlab/background_migration/link_lfs_objects.rb @@ -6,8 +6,6 @@ module Gitlab class LinkLfsObjects # Model definition used for migration class ForkNetworkMember < ActiveRecord::Base - include EachBatch - self.table_name = 'fork_network_members' def self.with_non_existing_lfs_objects @@ -25,62 +23,8 @@ module Gitlab end end - # Model definition used for migration - class Project < ActiveRecord::Base - include EachBatch - - self.table_name = 'projects' - - has_one :fork_network_member, class_name: 'LinkLfsObjects::ForkNetworkMember' - - def self.with_non_existing_lfs_objects - fork_network_members = - ForkNetworkMember.with_non_existing_lfs_objects - .select(1) - .where('fork_network_members.project_id = projects.id') - - where('EXISTS (?)', fork_network_members) - end - end - - # Model definition used for migration - class LfsObjectsProject < ActiveRecord::Base - include EachBatch - - self.table_name = 'lfs_objects_projects' - end - - BATCH_SIZE = 1000 - def perform(start_id, end_id) - forks = - Project - .with_non_existing_lfs_objects - .where(id: start_id..end_id) - - forks.includes(:fork_network_member).find_each do |project| - LfsObjectsProject - .select("lfs_objects_projects.lfs_object_id, #{project.id}, NOW(), NOW()") - .where(project_id: project.fork_network_member.forked_from_project_id) - .each_batch(of: BATCH_SIZE) do |batch| - execute <<~SQL - INSERT INTO lfs_objects_projects (lfs_object_id, project_id, created_at, updated_at) - #{batch.to_sql} - SQL - end - end - - logger.info(message: "LinkLfsObjects: created missing LfsObjectsProject for Projects #{forks.map(&:id).join(', ')}") - end - - private - - def execute(sql) - ::ActiveRecord::Base.connection.execute(sql) - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + # no-op as some queries times out end end end diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb index acfb641aeec..e58def57e69 100644 --- a/lib/gitlab/cycle_analytics/usage_data.rb +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -3,15 +3,32 @@ module Gitlab module CycleAnalytics class UsageData + include Gitlab::Utils::StrongMemoize PROJECTS_LIMIT = 10 - attr_reader :projects, :options + attr_reader :options def initialize - @projects = Project.sorted_by_activity.limit(PROJECTS_LIMIT) @options = { from: 7.days.ago } end + def projects + strong_memoize(:projects) do + projects = Project.where.not(last_activity_at: nil).order(last_activity_at: :desc).limit(10) + + Project.where.not(last_repository_updated_at: nil).order(last_repository_updated_at: :desc).limit(10) + + projects = projects.uniq.sort_by do |project| + [project.last_activity_at, project.last_repository_updated_at].min + end + + if projects.size < 10 + projects.concat(Project.where(last_activity_at: nil, last_repository_updated_at: nil).limit(10)) + end + + projects.uniq.first(10) + end + end + def to_json(*) total = 0 diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index 454dc778b6b..f11b7a0a298 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -2,6 +2,13 @@ module Gitlab module ImportExport - Error = Class.new(StandardError) + class Error < StandardError + def self.permission_error(user, importable) + self.new( + "User with ID: %s does not have required permissions for %s: %s with ID: %s" % + [user.id, importable.class.name, importable.name, importable.id] + ) + end + end end end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index e6f49dcac7a..cbaa6929efa 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -49,11 +49,7 @@ module Gitlab json = IO.read(@path) ActiveSupport::JSON.decode(json) rescue => e - @shared.logger.error( - group_id: @group.id, - group_name: @group.name, - message: "Import/Export error: #{e.message}" - ) + @shared.error(e) raise Gitlab::ImportExport::Error.new('Incorrect JSON format') end diff --git a/lib/gitlab/import_export/project/base_task.rb b/lib/gitlab/import_export/project/base_task.rb new file mode 100644 index 00000000000..6a7b24421c9 --- /dev/null +++ b/lib/gitlab/import_export/project/base_task.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class BaseTask + include Gitlab::WithRequestStore + + def initialize(opts, logger: Logger.new($stdout)) + @project_path = opts.fetch(:project_path) + @file_path = opts.fetch(:file_path) + @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) + @current_user = User.find_by_username(opts.fetch(:username)) + @measurement_enabled = opts.fetch(:measurement_enabled) + @measurement = Gitlab::Utils::Measuring.new(logger: logger) if @measurement_enabled + @logger = logger + end + + private + + attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path, :logger + + def measurement_enabled? + @measurement_enabled + end + + def success(message) + logger.info(message) + + true + end + + def error(message) + logger.error(message) + + false + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/export_task.rb b/lib/gitlab/import_export/project/export_task.rb new file mode 100644 index 00000000000..ec287380c48 --- /dev/null +++ b/lib/gitlab/import_export/project/export_task.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class ExportTask < BaseTask + def initialize(*) + super + + @project = namespace.projects.find_by_path(@project_path) + end + + def export + return error("Project with path: #{project_path} was not found. Please provide correct project path") unless project + return error("Invalid file path: #{file_path}. Please provide correct file path") unless file_path_exists? + + with_export do + ::Projects::ImportExport::ExportService.new(project, current_user) + .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path)) + end + + success('Done!') + end + + private + + def file_path_exists? + directory = File.dirname(file_path) + + Dir.exist?(directory) + end + + def with_export + with_request_store do + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + measurement_enabled? ? measurement.with_measuring { yield } : yield + end + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb new file mode 100644 index 00000000000..ae654ddbeaf --- /dev/null +++ b/lib/gitlab/import_export/project/import_task.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class ImportTask < BaseTask + def import + show_import_start_message + + run_isolated_sidekiq_job + + show_import_failures_count + + return error(project.import_state.last_error) if project.import_state&.last_error + return error(project.errors.full_messages.to_sentence) if project.errors.any? + + success('Done!') + end + + private + + # We want to ensure that all Sidekiq jobs are executed + # synchronously as part of that process. + # This ensures that all expensive operations do not escape + # to general Sidekiq clusters/nodes. + def with_isolated_sidekiq_job + Sidekiq::Testing.fake! do + with_request_store do + # If you are attempting to import a large project into a development environment, + # you may see Gitaly throw an error about too many calls or invocations. + # This is due to a n+1 calls limit being set for development setups (not enforced in production) + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 + # For development setups, this code-path will be excluded from n+1 detection. + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + measurement_enabled? ? measurement.with_measuring { yield } : yield + end + end + + true + end + end + + def run_isolated_sidekiq_job + with_isolated_sidekiq_job do + @project = create_project + + execute_sidekiq_job + end + end + + def create_project + # We are disabling ObjectStorage for `import` + # as it is too slow to handle big archives: + # 1. DB transaction timeouts on upload + # 2. Download of archive before unpacking + disable_upload_object_storage do + service = Projects::GitlabProjectsImportService.new( + current_user, + { + namespace_id: namespace.id, + path: project_path, + file: File.open(file_path) + } + ) + + service.execute + end + end + + def execute_sidekiq_job + Sidekiq::Worker.drain_all + end + + def disable_upload_object_storage + overwrite_uploads_setting('background_upload', false) do + overwrite_uploads_setting('direct_upload', false) do + yield + end + end + end + + def overwrite_uploads_setting(key, value) + old_value = Settings.uploads.object_store[key] + Settings.uploads.object_store[key] = value + + yield + + ensure + Settings.uploads.object_store[key] = old_value + end + + def full_path + "#{namespace.full_path}/#{project_path}" + end + + def show_import_start_message + logger.info "Importing GitLab export: #{file_path} into GitLab" \ + " #{full_path}" \ + " as #{current_user.name}" + end + + def show_import_failures_count + return unless project.import_failures.exists? + + logger.info "Total number of not imported relations: #{project.import_failures.count}" + end + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 8d81b2af065..09ed4eb568d 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -94,14 +94,6 @@ module Gitlab end end - def log_error(details) - @logger.error(log_base_data.merge(details)) - end - - def log_debug(details) - @logger.debug(log_base_data.merge(details)) - end - def log_base_data log = { importer: 'Import/Export', diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6e29a3e4cc4..09ea1c49c22 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -122,6 +122,8 @@ module Gitlab def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json + rescue ActiveRecord::StatementInvalid + { avg_cycle_analytics: {} } end def features_usage_data @@ -232,7 +234,7 @@ module Gitlab end def count(relation, column = nil, fallback: -1, batch: true) - if batch && Feature.enabled?(:usage_ping_batch_counter) + if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) Gitlab::Database::BatchCount.batch_count(relation, column) else relation.count @@ -242,7 +244,7 @@ module Gitlab end def distinct_count(relation, column = nil, fallback: -1, batch: true) - if batch && Feature.enabled?(:usage_ping_batch_counter) + if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) Gitlab::Database::BatchCount.batch_distinct_count(relation, column) else relation.distinct_count_by(column) diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index 20c57e777d8..c9e6cb9c039 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -59,14 +59,15 @@ module Gitlab end def duration_in_numbers(duration_in_seconds) + milliseconds = duration_in_seconds.in_milliseconds % 1.second.in_milliseconds seconds = duration_in_seconds % 1.minute minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute) hours = duration_in_seconds / 1.hour if hours == 0 - "%02d:%02d" % [minutes, seconds] + "%02d:%02d:%03d" % [minutes, seconds, milliseconds] else - "%02d:%02d:%02d" % [hours, minutes, seconds] + "%02d:%02d:%02d:%03d" % [hours, minutes, seconds, milliseconds] end end end diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake index 6cedf4e8cf1..ae54636fdd3 100644 --- a/lib/tasks/gitlab/import_export/export.rake +++ b/lib/tasks/gitlab/import_export/export.rake @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'gitlab/with_request_store' - # Export project to archive # # @example @@ -14,81 +12,36 @@ namespace :gitlab do # Load it here to avoid polluting Rake tasks with Sidekiq test warnings require 'sidekiq/testing' - warn_user_is_not_gitlab - - if ENV['IMPORT_DEBUG'].present? - ActiveRecord::Base.logger = Logger.new(STDOUT) - Gitlab::Metrics::Exporter::SidekiqExporter.instance.start - end - - GitlabProjectExport.new( - namespace_path: args.namespace_path, - project_path: args.project_path, - username: args.username, - file_path: args.archive_path, - measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled) - ).export - end - end -end - -class GitlabProjectExport - include Gitlab::WithRequestStore - - def initialize(opts) - @project_path = opts.fetch(:project_path) - @file_path = opts.fetch(:file_path) - @current_user = User.find_by_username(opts.fetch(:username)) - namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) - @project = namespace.projects.find_by_path(@project_path) - @measurement_enabled = opts.fetch(:measurement_enabled) - @measurable = Gitlab::Utils::Measuring.new if @measurement_enabled - end - - def export - validate_project - validate_file_path - - with_export do - ::Projects::ImportExport::ExportService.new(project, current_user) - .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path)) - end - - puts 'Done!' - rescue StandardError => e - puts "Exception: #{e.message}" - puts e.backtrace - exit 1 - end - - private - - attr_reader :measurable, :project, :current_user, :file_path, :project_path - - def validate_project - unless project - puts "Error: Project with path: #{project_path} was not found. Please provide correct project path" - exit 1 - end - end - - def validate_file_path - directory = File.dirname(file_path) - unless Dir.exist?(directory) - puts "Error: Invalid file path: #{file_path}. Please provide correct file path" - exit 1 - end - end - - def with_export - with_request_store do - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - measurement_enabled? ? measurable.with_measuring { yield } : yield + logger = Logger.new($stdout) + + begin + warn_user_is_not_gitlab + + if ENV['EXPORT_DEBUG'].present? + ActiveRecord::Base.logger = logger + Gitlab::Metrics::Exporter::SidekiqExporter.instance.start + logger.level = Logger::DEBUG + else + logger.level = Logger::INFO + end + + task = Gitlab::ImportExport::Project::ExportTask.new( + namespace_path: args.namespace_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled), + logger: logger + ) + + success = task.export + + exit(success) + rescue StandardError => e + logger.error "Exception: #{e.message}" + logger.debug e.backtrace + exit 1 end end end - - def measurement_enabled? - @measurement_enabled - end end diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index 4ed724a5c82..6e2d0e75da8 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'gitlab/with_request_store' - # Import large project archives # # This task: @@ -18,148 +16,36 @@ namespace :gitlab do # Load it here to avoid polluting Rake tasks with Sidekiq test warnings require 'sidekiq/testing' - warn_user_is_not_gitlab - - if ENV['IMPORT_DEBUG'].present? - ActiveRecord::Base.logger = Logger.new(STDOUT) - end - - GitlabProjectImport.new( - namespace_path: args.namespace_path, - project_path: args.project_path, - username: args.username, - file_path: args.archive_path, - measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled) - ).import - end - end -end - -class GitlabProjectImport - include Gitlab::WithRequestStore - - def initialize(opts) - @project_path = opts.fetch(:project_path) - @file_path = opts.fetch(:file_path) - @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) - @current_user = User.find_by_username(opts.fetch(:username)) - @measurement_enabled = opts.fetch(:measurement_enabled) - @measurement = Gitlab::Utils::Measuring.new if @measurement_enabled - end - - def import - show_import_start_message + logger = Logger.new($stdout) - run_isolated_sidekiq_job + begin + warn_user_is_not_gitlab - show_import_failures_count - - if project&.import_state&.last_error - puts "ERROR: #{project.import_state.last_error}" - exit 1 - elsif project.errors.any? - puts "ERROR: #{project.errors.full_messages.join(', ')}" - exit 1 - else - puts 'Done!' - end - rescue StandardError => e - puts "Exception: #{e.message}" - puts e.backtrace - exit 1 - end - - private - - attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path - - def measurement_enabled? - @measurement_enabled - end - - # We want to ensure that all Sidekiq jobs are executed - # synchronously as part of that process. - # This ensures that all expensive operations do not escape - # to general Sidekiq clusters/nodes. - def with_isolated_sidekiq_job - Sidekiq::Testing.fake! do - with_request_store do - # If you are attempting to import a large project into a development environment, - # you may see Gitaly throw an error about too many calls or invocations. - # This is due to a n+1 calls limit being set for development setups (not enforced in production) - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 - # For development setups, this code-path will be excluded from n+1 detection. - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - measurement_enabled? ? measurement.with_measuring { yield } : yield + if ENV['IMPORT_DEBUG'].present? + ActiveRecord::Base.logger = logger + Gitlab::Metrics::Exporter::SidekiqExporter.instance.start + logger.level = Logger::DEBUG + else + logger.level = Logger::INFO end - end - - true - end - end - - def run_isolated_sidekiq_job - with_isolated_sidekiq_job do - @project = create_project - - execute_sidekiq_job - end - end - - def create_project - # We are disabling ObjectStorage for `import` - # as it is too slow to handle big archives: - # 1. DB transaction timeouts on upload - # 2. Download of archive before unpacking - disable_upload_object_storage do - service = Projects::GitlabProjectsImportService.new( - current_user, - { - namespace_id: namespace.id, - path: project_path, - file: File.open(file_path) - } - ) - - service.execute - end - end - - def execute_sidekiq_job - Sidekiq::Worker.drain_all - end - def disable_upload_object_storage - overwrite_uploads_setting('background_upload', false) do - overwrite_uploads_setting('direct_upload', false) do - yield + task = Gitlab::ImportExport::Project::ImportTask.new( + namespace_path: args.namespace_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, + measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled), + logger: logger + ) + + success = task.import + + exit(success) + rescue StandardError => e + logger.error "Exception: #{e.message}" + logger.debug e.backtrace + exit 1 end end end - - def overwrite_uploads_setting(key, value) - old_value = Settings.uploads.object_store[key] - Settings.uploads.object_store[key] = value - - yield - - ensure - Settings.uploads.object_store[key] = old_value - end - - def full_path - "#{namespace.full_path}/#{project_path}" - end - - def show_import_start_message - puts "Importing GitLab export: #{file_path} into GitLab" \ - " #{full_path}" \ - " as #{current_user.name}" - end - - def show_import_failures_count - return unless project.import_failures.exists? - - puts "Total number of not imported relations: #{project.import_failures.count}" - end end |