From 810866ecb6c7be4fdac88dc3b2a6cd9ad49ac7bf Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 1 Jun 2017 15:27:35 +0100 Subject: backports changed import logic from pull mirroring feature into CE --- app/assets/javascripts/importer_status.js | 2 ++ app/controllers/projects/imports_controller.rb | 9 +----- app/models/application_setting.rb | 6 +--- app/models/project.rb | 29 +++++++++++++++++--- app/services/projects/create_service.rb | 5 ++-- app/workers/repository_fork_worker.rb | 38 ++++++++++++++------------ app/workers/repository_import_worker.rb | 24 ++++++++++++---- 7 files changed, 71 insertions(+), 42 deletions(-) (limited to 'app') diff --git a/app/assets/javascripts/importer_status.js b/app/assets/javascripts/importer_status.js index 34e4a257ff9..5b4ca94ed30 100644 --- a/app/assets/javascripts/importer_status.js +++ b/app/assets/javascripts/importer_status.js @@ -56,6 +56,8 @@ if (job.import_status === 'finished') { job_item.removeClass("active").addClass("success"); return status_field.html(' done'); + } else if (job.import_status === 'scheduled') { + return status_field.html(" scheduled"); } else if (job.import_status === 'started') { return status_field.html(" started"); } else { diff --git a/app/controllers/projects/imports_controller.rb b/app/controllers/projects/imports_controller.rb index a1b84afcd91..4b143434ea5 100644 --- a/app/controllers/projects/imports_controller.rb +++ b/app/controllers/projects/imports_controller.rb @@ -14,14 +14,7 @@ class Projects::ImportsController < Projects::ApplicationController @project.import_url = params[:project][:import_url] if @project.save - @project.reload - - if @project.import_failed? - @project.import_retry - else - @project.import_start - @project.add_import_job - end + @project.reload.import_schedule end redirect_to namespace_project_import_path(@project.namespace, @project) diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 3d12f3c306b..e72e581e580 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -199,7 +199,7 @@ class ApplicationSetting < ActiveRecord::Base ApplicationSetting.define_attribute_methods end - def self.defaults_ce + def self.defaults { after_sign_up_text: nil, akismet_enabled: false, @@ -250,10 +250,6 @@ class ApplicationSetting < ActiveRecord::Base } end - def self.defaults - defaults_ce - end - def self.create_from_defaults create(defaults) end diff --git a/app/models/project.rb b/app/models/project.rb index 7cb79e3249d..92229a3d9e0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -165,7 +165,7 @@ class Project < ActiveRecord::Base has_many :todos, dependent: :destroy has_many :notification_settings, dependent: :destroy, as: :source - has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" + has_one :import_data, dependent: :delete, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete has_many :container_repositories, dependent: :destroy @@ -298,8 +298,16 @@ class Project < ActiveRecord::Base scope :excluding_project, ->(project) { where.not(id: project) } state_machine :import_status, initial: :none do + event :import_schedule do + transition [:none, :finished, :failed] => :scheduled + end + + event :force_import_start do + transition [:none, :finished, :failed] => :started + end + event :import_start do - transition [:none, :finished] => :started + transition scheduled: :started end event :import_finish do @@ -307,18 +315,23 @@ class Project < ActiveRecord::Base end event :import_fail do - transition started: :failed + transition [:scheduled, :started] => :failed end event :import_retry do transition failed: :started end + state :scheduled state :started state :finished state :failed - after_transition any => :finished, do: :reset_cache_and_import_attrs + after_transition [:none, :finished, :failed] => :scheduled do |project, _| + project.run_after_commit { add_import_job } + end + + after_transition started: :finished, do: :reset_cache_and_import_attrs end class << self @@ -530,9 +543,17 @@ class Project < ActiveRecord::Base end def import_in_progress? + import_started? || import_scheduled? + end + + def import_started? import? && import_status == 'started' end + def import_scheduled? + import_status == 'scheduled' + end + def import_failed? import_status == 'failed' end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 535d93385e6..e874a2d8789 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -48,15 +48,14 @@ module Projects save_project_and_import_data(import_data) - @project.import_start if @project.import? - after_create_actions if @project.persisted? if @project.errors.empty? - @project.add_import_job if @project.import? + @project.import_schedule if @project.import? else fail(error: @project.errors.full_messages.join(', ')) end + @project rescue ActiveRecord::RecordInvalid => e message = "Unable to save #{e.record.type}: #{e.record.errors.full_messages.join(", ")} " diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index efc99ec962a..a338523dc6b 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -1,4 +1,6 @@ class RepositoryForkWorker + ForkError = Class.new(StandardError) + include Sidekiq::Worker include Gitlab::ShellAdapter include DedicatedSidekiqQueue @@ -8,29 +10,31 @@ class RepositoryForkWorker source_path: source_path, target_path: target_path) - project = Project.find_by_id(project_id) - - unless project.present? - logger.error("Project #{project_id} no longer exists!") - return - end + project = Project.find(project_id) + project.import_start result = gitlab_shell.fork_repository(forked_from_repository_storage_path, source_path, project.repository_storage_path, target_path) - unless result - logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") - project.mark_import_as_failed('The project could not be forked.') - return - end + raise ForkError, "Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}" unless result project.repository.after_import - - unless project.valid_repo? - logger.error("Project #{project_id} had an invalid repository after fork") - project.mark_import_as_failed('The forked repository is invalid.') - return - end + raise ForkError, "Project #{project_id} had an invalid repository after fork" unless project.valid_repo? project.import_finish + rescue ForkError => ex + fail_fork(project, ex.message) + raise + rescue => ex + return unless project + + fail_fork(project, ex.message) + raise ForkError, "#{ex.class} #{ex.message}" + end + + private + + def fail_fork(project, message) + Rails.logger.error(message) + project.mark_import_as_failed(message) end end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index b33ba2ed7c1..625476b7e01 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -1,4 +1,6 @@ class RepositoryImportWorker + ImportError = Class.new(StandardError) + include Sidekiq::Worker include DedicatedSidekiqQueue @@ -10,6 +12,8 @@ class RepositoryImportWorker @project = Project.find(project_id) @current_user = @project.creator + project.import_start + Gitlab::Metrics.add_event(:import_repository, import_url: @project.import_url, path: @project.path_with_namespace) @@ -17,13 +21,23 @@ class RepositoryImportWorker project.update_columns(import_jid: self.jid, import_error: nil) result = Projects::ImportService.new(project, current_user).execute - - if result[:status] == :error - project.mark_import_as_failed(result[:message]) - return - end + raise ImportError, result[:message] if result[:status] == :error project.repository.after_import project.import_finish + rescue ImportError => ex + fail_import(project, ex.message) + raise + rescue => ex + return unless project + + fail_import(project, ex.message) + raise ImportError, "#{ex.class} #{ex.message}" + end + + private + + def fail_import(project, message) + project.mark_import_as_failed(message) end end -- cgit v1.2.1