summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTiago Botelho <tiagonbotelho@hotmail.com>2018-05-03 13:55:14 +0100
committerTiago Botelho <tiagonbotelho@hotmail.com>2018-05-07 11:59:51 +0200
commit9a13059332a0c81b3a953f57bb9e40346eba951d (patch)
tree8f2806ec52b77cd212a19f9756c837606d988c62
parent2d84de9ec90b87ee4e1e10ce1a4a9d00b07c39e7 (diff)
downloadgitlab-ce-9a13059332a0c81b3a953f57bb9e40346eba951d.tar.gz
Backports every CE related change from ee-5484 to CE
-rw-r--r--app/controllers/projects/mirrors_controller.rb62
-rw-r--r--app/controllers/projects/settings/repository_controller.rb8
-rw-r--r--app/models/project.rb41
-rw-r--r--app/models/remote_mirror.rb216
-rw-r--r--app/models/repository.rb14
-rw-r--r--app/serializers/project_mirror_entity.rb13
-rw-r--r--app/services/git_push_service.rb8
-rw-r--r--app/services/projects/update_remote_mirror_service.rb30
-rw-r--r--app/views/projects/mirrors/_push.html.haml50
-rw-r--r--app/views/projects/mirrors/_show.html.haml3
-rw-r--r--app/views/shared/_remote_mirror_update_button.html.haml13
-rw-r--r--app/workers/all_queues.yml1
-rw-r--r--app/workers/repository_remove_remote_worker.rb35
-rw-r--r--app/workers/repository_update_remote_mirror_worker.rb49
-rw-r--r--config/sidekiq_queues.yml3
-rw-r--r--db/schema.rb6
-rw-r--r--lib/gitlab/import_export/import_export.yml1
-rw-r--r--lib/gitlab/usage_data.rb1
-rw-r--r--spec/controllers/projects/mirrors_controller_spec.rb72
-rw-r--r--spec/factories/projects.rb11
-rw-r--r--spec/factories/remote_mirrors.rb6
-rw-r--r--spec/features/projects/remote_mirror_spec.rb34
-rw-r--r--spec/features/projects/settings/repository_settings_spec.rb15
-rw-r--r--spec/lib/gitlab/import_export/all_models.yml1
-rw-r--r--spec/lib/gitlab/usage_data_spec.rb1
-rw-r--r--spec/models/project_spec.rb79
-rw-r--r--spec/models/remote_mirror_spec.rb275
-rw-r--r--spec/models/repository_spec.rb32
-rw-r--r--spec/services/git_push_service_spec.rb66
-rw-r--r--spec/services/projects/destroy_service_spec.rb13
-rw-r--r--spec/services/projects/update_remote_mirror_service_spec.rb355
-rw-r--r--spec/workers/repository_remove_remote_worker_spec.rb50
-rw-r--r--spec/workers/repository_update_remote_mirror_worker_spec.rb84
33 files changed, 1645 insertions, 3 deletions
diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb
new file mode 100644
index 00000000000..fb5aee2e702
--- /dev/null
+++ b/app/controllers/projects/mirrors_controller.rb
@@ -0,0 +1,62 @@
+class Projects::MirrorsController < Projects::ApplicationController
+ include RepositorySettingsRedirect
+
+ # Authorize
+ before_action :authorize_admin_mirror!
+ before_action :remote_mirror, only: [:update]
+
+ layout "project_settings"
+
+ def show
+ redirect_to_repository_settings(project)
+ end
+
+ def update
+ if project.update_attributes(mirror_params)
+ flash[:notice] = 'Mirroring settings were successfully updated.'
+ else
+ flash[:alert] = project.errors.full_messages.join(', ').html_safe
+ end
+
+ respond_to do |format|
+ format.html { redirect_to_repository_settings(project) }
+ format.json do
+ if project.errors.present?
+ render json: project.errors, status: :unprocessable_entity
+ else
+ render json: ProjectMirrorSerializer.new.represent(project)
+ end
+ end
+ end
+ end
+
+ def update_now
+ if params[:sync_remote]
+ project.update_remote_mirrors
+ flash[:notice] = "The remote repository is being updated..."
+ end
+
+ redirect_to_repository_settings(project)
+ end
+
+ private
+
+ def remote_mirror
+ @remote_mirror = project.remote_mirrors.first_or_initialize
+ end
+
+ def mirror_params_attributes
+ [
+ remote_mirrors_attributes: %i[
+ url
+ id
+ enabled
+ only_protected_branches
+ ]
+ ]
+ end
+
+ def mirror_params
+ params.require(:project).permit(mirror_params_attributes)
+ end
+end
diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb
index f17056f13e0..a3bb60bb3b2 100644
--- a/app/controllers/projects/settings/repository_controller.rb
+++ b/app/controllers/projects/settings/repository_controller.rb
@@ -2,6 +2,7 @@ module Projects
module Settings
class RepositoryController < Projects::ApplicationController
before_action :authorize_admin_project!
+ before_action :remote_mirror, only: [:show]
def show
render_show
@@ -25,6 +26,7 @@ module Projects
define_deploy_token
define_protected_refs
+ remote_mirror
render 'show'
end
@@ -41,6 +43,12 @@ module Projects
load_gon_index
end
+ def remote_mirror
+ return unless project.feature_available?(:repository_mirrors)
+
+ @remote_mirror = project.remote_mirrors.first_or_initialize
+ end
+
def access_levels_options
{
create_access_levels: levels_for_dropdown,
diff --git a/app/models/project.rb b/app/models/project.rb
index b12b694aabd..4021e4e85bf 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -64,6 +64,9 @@ class Project < ActiveRecord::Base
default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
add_authentication_token_field :runners_token
+
+ before_validation :mark_remote_mirrors_for_removal
+
before_save :ensure_runners_token
after_save :update_project_statistics, if: :namespace_id_changed?
@@ -241,11 +244,17 @@ class Project < ActiveRecord::Base
has_many :project_badges, class_name: 'ProjectBadge'
has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+ has_many :remote_mirrors, inverse_of: :project
+
accepts_nested_attributes_for :variables, allow_destroy: true
accepts_nested_attributes_for :project_feature, update_only: true
accepts_nested_attributes_for :import_data
accepts_nested_attributes_for :auto_devops, update_only: true
+ accepts_nested_attributes_for :remote_mirrors,
+ allow_destroy: true,
+ reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
+
delegate :name, to: :owner, allow_nil: true, prefix: true
delegate :members, to: :team, prefix: true
delegate :add_user, :add_users, to: :team
@@ -335,6 +344,7 @@ class Project < ActiveRecord::Base
scope :with_issues_enabled, -> { with_feature_enabled(:issues) }
scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
+ scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
scope :with_group_runners_enabled, -> do
joins(:ci_cd_settings)
@@ -754,6 +764,37 @@ class Project < ActiveRecord::Base
import_type == 'gitea'
end
+ def has_remote_mirror?
+ remote_mirror_available? && remote_mirrors.enabled.exists?
+ end
+
+ def updating_remote_mirror?
+ remote_mirrors.enabled.started.exists?
+ end
+
+ def update_remote_mirrors
+ return unless remote_mirror_available?
+
+ remote_mirrors.enabled.each(&:sync)
+ end
+
+ def mark_stuck_remote_mirrors_as_failed!
+ remote_mirrors.stuck.update_all(
+ update_status: :failed,
+ last_error: 'The remote mirror took to long to complete.',
+ last_update_at: Time.now
+ )
+ end
+
+ def mark_remote_mirrors_for_removal
+ remote_mirrors.each(&:mark_for_delete_if_blank_url)
+ end
+
+ def remote_mirror_available?
+ remote_mirror_available_overridden ||
+ ::Gitlab::CurrentSettings.mirror_available
+ end
+
def check_limit
unless creator.can_create_project? || namespace.kind == 'group'
projects_limit = creator.projects_limit
diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb
new file mode 100644
index 00000000000..b7e45875830
--- /dev/null
+++ b/app/models/remote_mirror.rb
@@ -0,0 +1,216 @@
+class RemoteMirror < ActiveRecord::Base
+ include AfterCommitQueue
+
+ PROTECTED_BACKOFF_DELAY = 1.minute
+ UNPROTECTED_BACKOFF_DELAY = 5.minutes
+
+ attr_encrypted :credentials,
+ key: Gitlab::Application.secrets.db_key_base,
+ marshal: true,
+ encode: true,
+ mode: :per_attribute_iv_and_salt,
+ insecure_mode: true,
+ algorithm: 'aes-256-cbc'
+
+ default_value_for :only_protected_branches, true
+
+ belongs_to :project, inverse_of: :remote_mirrors
+
+ validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
+ validates :url, addressable_url: true, if: :url_changed?
+
+ before_save :set_new_remote_name, if: :mirror_url_changed?
+
+ after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
+ after_save :refresh_remote, if: :mirror_url_changed?
+ after_update :reset_fields, if: :mirror_url_changed?
+
+ after_commit :remove_remote, on: :destroy
+
+ scope :enabled, -> { where(enabled: true) }
+ scope :started, -> { with_update_status(:started) }
+ scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
+
+ state_machine :update_status, initial: :none do
+ event :update_start do
+ transition [:none, :finished, :failed] => :started
+ end
+
+ event :update_finish do
+ transition started: :finished
+ end
+
+ event :update_fail do
+ transition started: :failed
+ end
+
+ state :started
+ state :finished
+ state :failed
+
+ after_transition any => :started do |remote_mirror, _|
+ Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path)
+
+ remote_mirror.update(last_update_started_at: Time.now)
+ end
+
+ after_transition started: :finished do |remote_mirror, _|
+ Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path)
+
+ timestamp = Time.now
+ remote_mirror.update_attributes!(
+ last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
+ )
+ end
+
+ after_transition started: :failed do |remote_mirror, _|
+ Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path)
+
+ remote_mirror.update(last_update_at: Time.now)
+ end
+ end
+
+ def remote_name
+ super || fallback_remote_name
+ end
+
+ def update_failed?
+ update_status == 'failed'
+ end
+
+ def update_in_progress?
+ update_status == 'started'
+ end
+
+ def update_repository(options)
+ raw.update(options)
+ end
+
+ def sync
+ return unless enabled?
+ return if Gitlab::Geo.secondary?
+
+ if recently_scheduled?
+ RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
+ else
+ RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
+ end
+ end
+
+ def enabled
+ return false unless project && super
+ return false unless project.remote_mirror_available?
+ return false unless project.repository_exists?
+ return false if project.pending_delete?
+
+ true
+ end
+ alias_method :enabled?, :enabled
+
+ def updated_since?(timestamp)
+ last_update_started_at && last_update_started_at > timestamp && !update_failed?
+ end
+
+ def mark_for_delete_if_blank_url
+ mark_for_destruction if url.blank?
+ end
+
+ def mark_as_failed(error_message)
+ update_fail
+ update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
+ end
+
+ def url=(value)
+ super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
+
+ mirror_url = Gitlab::UrlSanitizer.new(value)
+ self.credentials = mirror_url.credentials
+
+ super(mirror_url.sanitized_url)
+ end
+
+ def url
+ if super
+ Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
+ end
+ rescue
+ super
+ end
+
+ def safe_url
+ return if url.nil?
+
+ result = URI.parse(url)
+ result.password = '*****' if result.password
+ result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
+ result.to_s
+ end
+
+ private
+
+ def raw
+ @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
+ end
+
+ def fallback_remote_name
+ return unless id
+
+ "remote_mirror_#{id}"
+ end
+
+ def recently_scheduled?
+ return false unless self.last_update_started_at
+
+ self.last_update_started_at >= Time.now - backoff_delay
+ end
+
+ def backoff_delay
+ if self.only_protected_branches
+ PROTECTED_BACKOFF_DELAY
+ else
+ UNPROTECTED_BACKOFF_DELAY
+ end
+ end
+
+ def reset_fields
+ update_columns(
+ last_error: nil,
+ last_update_at: nil,
+ last_successful_update_at: nil,
+ update_status: 'finished'
+ )
+ end
+
+ def set_override_remote_mirror_available
+ enabled = read_attribute(:enabled)
+
+ project.update(remote_mirror_available_overridden: enabled)
+ end
+
+ def set_new_remote_name
+ self.remote_name = "remote_mirror_#{SecureRandom.hex}"
+ end
+
+ def refresh_remote
+ return unless project
+
+ # Before adding a new remote we have to delete the data from
+ # the previous remote name
+ prev_remote_name = remote_name_was || fallback_remote_name
+ run_after_commit do
+ project.repository.async_remove_remote(prev_remote_name)
+ end
+
+ project.repository.add_remote(remote_name, url)
+ end
+
+ def remove_remote
+ return unless project # could be pending to delete so don't need to touch the git repository
+
+ project.repository.async_remove_remote(remote_name)
+ end
+
+ def mirror_url_changed?
+ url_changed? || encrypted_credentials_changed?
+ end
+end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 6831305fb93..2a0802482f5 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -861,6 +861,20 @@ class Repository
gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
end
+ def async_remove_remote(remote_name)
+ return unless remote_name
+
+ job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name)
+
+ if job_id
+ Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.")
+ else
+ Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.")
+ end
+
+ job_id
+ end
+
def fetch_source_branch!(source_repository, source_branch, local_ref)
raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
end
diff --git a/app/serializers/project_mirror_entity.rb b/app/serializers/project_mirror_entity.rb
new file mode 100644
index 00000000000..8051f6e7d35
--- /dev/null
+++ b/app/serializers/project_mirror_entity.rb
@@ -0,0 +1,13 @@
+class ProjectMirrorEntity < Grape::Entity
+ prepend ::EE::ProjectMirrorEntity
+
+ expose :id
+
+ expose :remote_mirrors_attributes do |project|
+ next [] unless project.remote_mirrors.present?
+
+ project.remote_mirrors.map do |remote|
+ remote.as_json(only: %i[id url enabled])
+ end
+ end
+end
diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index c037141fcde..f3bfc53dcd3 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -55,6 +55,7 @@ class GitPushService < BaseService
execute_related_hooks
perform_housekeeping
+ update_remote_mirrors
update_caches
update_signatures
@@ -119,6 +120,13 @@ class GitPushService < BaseService
protected
+ def update_remote_mirrors
+ return unless @project.has_remote_mirror?
+
+ @project.mark_stuck_remote_mirrors_as_failed!
+ @project.update_remote_mirrors
+ end
+
def execute_related_hooks
# Update merge requests that may be affected by this push. A new branch
# could cause the last commit of a merge request to change.
diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb
new file mode 100644
index 00000000000..8183a2f26d7
--- /dev/null
+++ b/app/services/projects/update_remote_mirror_service.rb
@@ -0,0 +1,30 @@
+module Projects
+ class UpdateRemoteMirrorService < BaseService
+ attr_reader :errors
+
+ def execute(remote_mirror)
+ @errors = []
+
+ return success unless remote_mirror.enabled?
+
+ begin
+ repository.fetch_remote(remote_mirror.remote_name, no_tags: true)
+
+ opts = {}
+ if remote_mirror.only_protected_branches?
+ opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
+ end
+
+ remote_mirror.update_repository(opts)
+ rescue => e
+ errors << e.message.strip
+ end
+
+ if errors.present?
+ error(errors.join("\n\n"))
+ else
+ success
+ end
+ end
+ end
+end
diff --git a/app/views/projects/mirrors/_push.html.haml b/app/views/projects/mirrors/_push.html.haml
new file mode 100644
index 00000000000..4a6aefce351
--- /dev/null
+++ b/app/views/projects/mirrors/_push.html.haml
@@ -0,0 +1,50 @@
+- expanded = Rails.env.test?
+%section.settings.no-animate{ class: ('expanded' if expanded) }
+ .settings-header
+ %h4
+ Push to a remote repository
+ %button.btn.js-settings-toggle
+ = expanded ? 'Collapse' : 'Expand'
+ %p
+ Set up the remote repository that you want to update with the content of the current repository
+ every time someone pushes to it.
+ = link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
+ .settings-content
+ = form_for @project, url: project_mirror_path(@project) do |f|
+ %div
+ = form_errors(@project)
+ = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
+ - if @remote_mirror.last_error.present?
+ .panel.panel-danger
+ .panel-heading
+ - if @remote_mirror.last_update_at
+ The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
+ - else
+ The remote repository failed to update.
+
+ - if @remote_mirror.last_successful_update_at
+ Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
+ .panel-body
+ %pre
+ :preserve
+ #{h(@remote_mirror.last_error.strip)}
+ = f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
+ .form-group
+ = rm_form.check_box :enabled, class: "pull-left"
+ .prepend-left-20
+ = rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
+ %p.light.append-bottom-0
+ Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
+ .form-group.has-feedback
+ = rm_form.label :url, "Git repository URL", class: "label-light"
+ = rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
+
+ = render "projects/mirrors/instructions"
+
+ .form-group
+ = rm_form.check_box :only_protected_branches, class: 'pull-left'
+ .prepend-left-20
+ = rm_form.label :only_protected_branches, class: 'label-light'
+ = link_to icon('question-circle'), help_page_path('user/project/protected_branches')
+
+ = f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
diff --git a/app/views/projects/mirrors/_show.html.haml b/app/views/projects/mirrors/_show.html.haml
new file mode 100644
index 00000000000..da789906aef
--- /dev/null
+++ b/app/views/projects/mirrors/_show.html.haml
@@ -0,0 +1,3 @@
+- if can?(current_user, :admin_mirror, @project)
+ = render 'projects/mirrors/push'
+
diff --git a/app/views/shared/_remote_mirror_update_button.html.haml b/app/views/shared/_remote_mirror_update_button.html.haml
new file mode 100644
index 00000000000..34de1c0695f
--- /dev/null
+++ b/app/views/shared/_remote_mirror_update_button.html.haml
@@ -0,0 +1,13 @@
+- if @project.has_remote_mirror?
+ .append-bottom-default
+ - if remote_mirror.update_in_progress?
+ %span.btn.disabled
+ = icon("refresh spin")
+ Updating&hellip;
+ - else
+ = link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do
+ = icon("refresh")
+ Update Now
+ - if @remote_mirror.last_successful_update_at
+ %p.inline.prepend-left-10
+ Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index c469aea7052..d41bcd1abcb 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -112,3 +112,4 @@
- update_user_activity
- upload_checksum
- web_hook
+- repository_update_remote_mirror
diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb
new file mode 100644
index 00000000000..1c19b604b77
--- /dev/null
+++ b/app/workers/repository_remove_remote_worker.rb
@@ -0,0 +1,35 @@
+class RepositoryRemoveRemoteWorker
+ include ApplicationWorker
+ include ExclusiveLeaseGuard
+
+ LEASE_TIMEOUT = 1.hour
+
+ attr_reader :project, :remote_name
+
+ def perform(project_id, remote_name)
+ @remote_name = remote_name
+ @project = Project.find_by_id(project_id)
+
+ return unless @project
+
+ logger.info("Removing remote #{remote_name} from project #{project.id}")
+
+ try_obtain_lease do
+ remove_remote = @project.repository.remove_remote(remote_name)
+
+ if remove_remote
+ logger.info("Remote #{remote_name} was successfully removed from project #{project.id}")
+ else
+ logger.error("Could not remove remote #{remote_name} from project #{project.id}")
+ end
+ end
+ end
+
+ def lease_timeout
+ LEASE_TIMEOUT
+ end
+
+ def lease_key
+ "remove_remote_#{project.id}_#{remote_name}"
+ end
+end
diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb
new file mode 100644
index 00000000000..bb963979e88
--- /dev/null
+++ b/app/workers/repository_update_remote_mirror_worker.rb
@@ -0,0 +1,49 @@
+class RepositoryUpdateRemoteMirrorWorker
+ UpdateAlreadyInProgressError = Class.new(StandardError)
+ UpdateError = Class.new(StandardError)
+
+ include ApplicationWorker
+ include Gitlab::ShellAdapter
+
+ sidekiq_options retry: 3, dead: false
+
+ sidekiq_retry_in { |count| 30 * count }
+
+ sidekiq_retries_exhausted do |msg, _|
+ Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
+ end
+
+ def perform(remote_mirror_id, scheduled_time)
+ remote_mirror = RemoteMirror.find(remote_mirror_id)
+ return if remote_mirror.updated_since?(scheduled_time)
+
+ raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
+
+ remote_mirror.update_start
+
+ project = remote_mirror.project
+ current_user = project.creator
+ result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
+ raise UpdateError, result[:message] if result[:status] == :error
+
+ remote_mirror.update_finish
+ rescue UpdateAlreadyInProgressError
+ raise
+ rescue UpdateError => ex
+ fail_remote_mirror(remote_mirror, ex.message)
+ raise
+ rescue => ex
+ return unless remote_mirror
+
+ fail_remote_mirror(remote_mirror, ex.message)
+ raise UpdateError, "#{ex.class}: #{ex.message}"
+ end
+
+ private
+
+ def fail_remote_mirror(remote_mirror, message)
+ remote_mirror.mark_as_failed(message)
+
+ Rails.logger.error(message)
+ end
+end
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 47fbbed44cf..e1e8f36b663 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -73,3 +73,6 @@
- [object_storage, 1]
- [plugin, 1]
- [pipeline_background, 1]
+ - [repository_update_remote_mirror, 1]
+ - [repository_remove_remote, 1]
+
diff --git a/db/schema.rb b/db/schema.rb
index 1f592c019fa..f03ece6d75f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -314,10 +314,10 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.integer "auto_canceled_by_id"
t.boolean "retried"
t.integer "stage_id"
- t.integer "artifacts_file_store"
- t.integer "artifacts_metadata_store"
t.boolean "protected"
t.integer "failure_reason"
+ t.integer "artifacts_file_store"
+ t.integer "artifacts_metadata_store"
end
add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
@@ -365,13 +365,13 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.integer "project_id", null: false
t.integer "job_id", null: false
t.integer "file_type", null: false
- t.integer "file_store"
t.integer "size", limit: 8
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.datetime_with_timezone "expire_at"
t.string "file"
t.binary "file_sha256"
+ t.integer "file_store"
end
add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 0d1c4f73c6e..21ac7f7e0b6 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -106,6 +106,7 @@ excluded_attributes:
- :last_repository_updated_at
- :last_repository_check_at
- :storage_version
+ - :remote_mirror_available_overridden
- :description_html
snippets:
- :expired_at
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 8c0a4d55ea2..e294f3c4ebc 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -71,6 +71,7 @@ module Gitlab
projects_imported_from_github: Project.where(import_type: 'github').count,
protected_branches: ProtectedBranch.count,
releases: Release.count,
+ remote_mirrors: RemoteMirror.count,
snippets: Snippet.count,
todos: Todo.count,
uploads: Upload.count,
diff --git a/spec/controllers/projects/mirrors_controller_spec.rb b/spec/controllers/projects/mirrors_controller_spec.rb
new file mode 100644
index 00000000000..45c1218a39c
--- /dev/null
+++ b/spec/controllers/projects/mirrors_controller_spec.rb
@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe Projects::MirrorsController do
+ include ReactiveCachingHelpers
+
+ describe 'setting up a remote mirror' do
+ set(:project) { create(:project, :repository) }
+
+ context 'when the current project is not a mirror' do
+ it 'allows to create a remote mirror' do
+ sign_in(project.owner)
+
+ expect do
+ do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } })
+ end.to change { RemoteMirror.count }.to(1)
+ end
+ end
+ end
+
+ describe '#update' do
+ let(:project) { create(:project, :repository, :remote_mirror) }
+
+ before do
+ sign_in(project.owner)
+ end
+
+ around do |example|
+ Sidekiq::Testing.fake! { example.run }
+ end
+
+ context 'With valid URL for a push' do
+ let(:remote_mirror_attributes) do
+ { "0" => { "enabled" => "0", url: 'https://updated.example.com' } }
+ end
+
+ it 'processes a successful update' do
+ do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+ expect(response).to redirect_to(project_settings_repository_path(project))
+ expect(flash[:notice]).to match(/successfully updated/)
+ end
+
+ it 'should create a RemoteMirror object' do
+ expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.to change(RemoteMirror, :count).by(1)
+ end
+ end
+
+ context 'With invalid URL for a push' do
+ let(:remote_mirror_attributes) do
+ { "0" => { "enabled" => "0", url: 'ftp://invalid.invalid' } }
+ end
+
+ it 'processes an unsuccessful update' do
+ do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+ expect(response).to redirect_to(project_settings_repository_path(project))
+ expect(flash[:alert]).to match(/must be a valid URL/)
+ end
+
+ it 'should not create a RemoteMirror object' do
+ expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.not_to change(RemoteMirror, :count)
+ end
+ end
+ end
+
+ def do_put(project, options, extra_attrs = {})
+ attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
+ attrs[:project] = options
+
+ put :update, attrs
+ end
+end
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index a6128903546..a24ecec6c2e 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -183,6 +183,17 @@ FactoryBot.define do
end
end
+ trait :remote_mirror do
+ transient do
+ remote_name "remote_mirror_#{SecureRandom.hex}"
+ url "http://foo.com"
+ enabled true
+ end
+ after(:create) do |project, evaluator|
+ project.remote_mirrors.create!(url: evaluator.url, enabled: evaluator.enabled)
+ end
+ end
+
trait :stubbed_repository do
after(:build) do |project|
allow(project).to receive(:empty_repo?).and_return(false)
diff --git a/spec/factories/remote_mirrors.rb b/spec/factories/remote_mirrors.rb
new file mode 100644
index 00000000000..adc7da27522
--- /dev/null
+++ b/spec/factories/remote_mirrors.rb
@@ -0,0 +1,6 @@
+FactoryBot.define do
+ factory :remote_mirror, class: 'RemoteMirror' do
+ association :project, :repository
+ url "http://foo:bar@test.com"
+ end
+end
diff --git a/spec/features/projects/remote_mirror_spec.rb b/spec/features/projects/remote_mirror_spec.rb
new file mode 100644
index 00000000000..81a6b613cc8
--- /dev/null
+++ b/spec/features/projects/remote_mirror_spec.rb
@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+feature 'Project remote mirror', :feature do
+ let(:project) { create(:project, :repository, :remote_mirror) }
+ let(:remote_mirror) { project.remote_mirrors.first }
+ let(:user) { create(:user) }
+
+ describe 'On a project', :js do
+ before do
+ project.add_master(user)
+ sign_in user
+ end
+
+ context 'when last_error is present but last_update_at is not' do
+ it 'renders error message without timstamp' do
+ remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: nil)
+
+ visit project_mirror_path(project)
+
+ expect(page).to have_content('The remote repository failed to update.')
+ end
+ end
+
+ context 'when last_error and last_update_at are present' do
+ it 'renders error message with timestamp' do
+ remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: Time.now - 5.minutes)
+
+ visit project_mirror_path(project)
+
+ expect(page).to have_content('The remote repository failed to update 5 minutes ago.')
+ end
+ end
+ end
+end
diff --git a/spec/features/projects/settings/repository_settings_spec.rb b/spec/features/projects/settings/repository_settings_spec.rb
index e1dfe617691..162aee63942 100644
--- a/spec/features/projects/settings/repository_settings_spec.rb
+++ b/spec/features/projects/settings/repository_settings_spec.rb
@@ -115,5 +115,20 @@ describe 'Projects > Settings > Repository settings' do
expect(page).to have_content('Your new project deploy token has been created')
end
end
+
+ context 'remote mirror settings' do
+ let(:user2) { create(:user) }
+
+ before do
+ project.add_master(user2)
+
+ visit project_settings_repository_path(project)
+ end
+
+ it 'shows push mirror settings' do
+ expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
+ expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')
+ end
+ end
end
end
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index 830d91de983..62b6a415f9e 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -268,6 +268,7 @@ project:
- pages_domains
- authorized_users
- project_authorizations
+- remote_mirrors
- route
- redirect_routes
- statistics
diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb
index 9e6aa109a4b..a716e6f5434 100644
--- a/spec/lib/gitlab/usage_data_spec.rb
+++ b/spec/lib/gitlab/usage_data_spec.rb
@@ -96,6 +96,7 @@ describe Gitlab::UsageData do
pages_domains
protected_branches
releases
+ remote_mirrors
snippets
todos
uploads
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index f3cf21cf279..8015c0926b6 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -1852,6 +1852,85 @@ describe Project do
it { expect(project.gitea_import?).to be true }
end
+ describe '#has_remote_mirror?' do
+ let(:project) { create(:project, :remote_mirror, :import_started) }
+ subject { project.has_remote_mirror? }
+
+ before do
+ allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+ end
+
+ it 'returns true when a remote mirror is enabled' do
+ is_expected.to be_truthy
+ end
+
+ it 'returns false when remote mirror is disabled' do
+ project.remote_mirrors.first.update_attributes(enabled: false)
+
+ is_expected.to be_falsy
+ end
+ end
+
+ describe '#update_remote_mirrors' do
+ let(:project) { create(:project, :remote_mirror, :import_started) }
+ delegate :update_remote_mirrors, to: :project
+
+ before do
+ allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+ end
+
+ it 'syncs enabled remote mirror' do
+ expect_any_instance_of(RemoteMirror).to receive(:sync)
+
+ update_remote_mirrors
+ end
+
+ # TODO: study if remote_mirror_available_overridden is still a necessary attribute considering that
+ # it is no longer under any license
+ it 'does nothing when remote mirror is disabled globally and not overridden' do
+ stub_application_setting(mirror_available: false)
+ project.remote_mirror_available_overridden = false
+
+ expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+ update_remote_mirrors
+ end
+
+ it 'does not sync disabled remote mirrors' do
+ project.remote_mirrors.first.update_attributes(enabled: false)
+
+ expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+ update_remote_mirrors
+ end
+ end
+
+ describe '#remote_mirror_available?' do
+ let(:project) { create(:project) }
+
+ context 'when remote mirror global setting is enabled' do
+ it 'returns true' do
+ expect(project.remote_mirror_available?).to be(true)
+ end
+ end
+
+ context 'when remote mirror global setting is disabled' do
+ before do
+ stub_application_setting(mirror_available: false)
+ end
+
+ it 'returns true when overridden' do
+ project.remote_mirror_available_overridden = true
+
+ expect(project.remote_mirror_available?).to be(true)
+ end
+
+ it 'returns false when not overridden' do
+ expect(project.remote_mirror_available?).to be(false)
+ end
+ end
+ end
+
describe '#ancestors_upto', :nested_groups do
let(:parent) { create(:group) }
let(:child) { create(:group, parent: parent) }
diff --git a/spec/models/remote_mirror_spec.rb b/spec/models/remote_mirror_spec.rb
new file mode 100644
index 00000000000..d6181ca0783
--- /dev/null
+++ b/spec/models/remote_mirror_spec.rb
@@ -0,0 +1,275 @@
+require 'rails_helper'
+
+describe RemoteMirror do
+ describe 'URL validation' do
+ context 'with a valid URL' do
+ it 'should be valid' do
+ remote_mirror = build(:remote_mirror)
+ expect(remote_mirror).to be_valid
+ end
+ end
+
+ context 'with an invalid URL' do
+ it 'should not be valid' do
+ remote_mirror = build(:remote_mirror, url: 'ftp://invalid.invalid')
+ expect(remote_mirror).not_to be_valid
+ expect(remote_mirror.errors[:url].size).to eq(2)
+ end
+ end
+ end
+
+ describe 'encrypting credentials' do
+ context 'when setting URL for a first time' do
+ it 'stores the URL without credentials' do
+ mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+ expect(mirror.read_attribute(:url)).to eq('http://test.com')
+ end
+
+ it 'stores the credentials on a separate field' do
+ mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+ expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
+ end
+
+ it 'handles credentials with large content' do
+ mirror = create_mirror(url: 'http://bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif:9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75@test.com')
+
+ expect(mirror.credentials).to eq({
+ user: 'bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif',
+ password: '9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75'
+ })
+ end
+ end
+
+ context 'when updating the URL' do
+ it 'allows a new URL without credentials' do
+ mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+ mirror.update_attribute(:url, 'http://test.com')
+
+ expect(mirror.url).to eq('http://test.com')
+ expect(mirror.credentials).to eq({ user: nil, password: nil })
+ end
+
+ it 'allows a new URL with credentials' do
+ mirror = create_mirror(url: 'http://test.com')
+
+ mirror.update_attribute(:url, 'http://foo:bar@test.com')
+
+ expect(mirror.url).to eq('http://foo:bar@test.com')
+ expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
+ end
+
+ it 'updates the remote config if credentials changed' do
+ mirror = create_mirror(url: 'http://foo:bar@test.com')
+ repo = mirror.project.repository
+
+ mirror.update_attribute(:url, 'http://foo:baz@test.com')
+
+ config = repo.raw_repository.rugged.config
+ expect(config["remote.#{mirror.remote_name}.url"]).to eq('http://foo:baz@test.com')
+ end
+
+ it 'removes previous remote' do
+ mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+ expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
+
+ mirror.update_attributes(url: 'http://test.com')
+ end
+ end
+ end
+
+ describe '#remote_name' do
+ context 'when remote name is persisted in the database' do
+ it 'returns remote name with random value' do
+ allow(SecureRandom).to receive(:hex).and_return('secret')
+
+ remote_mirror = create(:remote_mirror)
+
+ expect(remote_mirror.remote_name).to eq("remote_mirror_secret")
+ end
+ end
+
+ context 'when remote name is not persisted in the database' do
+ it 'returns remote name with remote mirror id' do
+ remote_mirror = create(:remote_mirror)
+ remote_mirror.remote_name = nil
+
+ expect(remote_mirror.remote_name).to eq("remote_mirror_#{remote_mirror.id}")
+ end
+ end
+
+ context 'when remote is not persisted in the database' do
+ it 'returns nil' do
+ remote_mirror = build(:remote_mirror, remote_name: nil)
+
+ expect(remote_mirror.remote_name).to be_nil
+ end
+ end
+ end
+
+ describe '#safe_url' do
+ context 'when URL contains credentials' do
+ it 'masks the credentials' do
+ mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+ expect(mirror.safe_url).to eq('http://*****:*****@test.com')
+ end
+ end
+
+ context 'when URL does not contain credentials' do
+ it 'shows the full URL' do
+ mirror = create_mirror(url: 'http://test.com')
+
+ expect(mirror.safe_url).to eq('http://test.com')
+ end
+ end
+ end
+
+ context 'when remote mirror gets destroyed' do
+ it 'removes remote' do
+ mirror = create_mirror(url: 'http://foo:bar@test.com')
+
+ expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
+
+ mirror.destroy!
+ end
+ end
+
+ context 'stuck mirrors' do
+ it 'includes mirrors stuck in started with no last_update_at set' do
+ mirror = create_mirror(url: 'http://cantbeblank',
+ update_status: 'started',
+ last_update_at: nil,
+ updated_at: 25.hours.ago)
+
+ expect(described_class.stuck.last).to eq(mirror)
+ end
+ end
+
+ context '#sync' do
+ let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+
+ around do |example|
+ Timecop.freeze { example.run }
+ end
+
+ context 'with remote mirroring disabled' do
+ it 'returns nil' do
+ remote_mirror.update_attributes(enabled: false)
+
+ expect(remote_mirror.sync).to be_nil
+ end
+ end
+
+ context 'as a Geo secondary' do
+ it 'returns nil' do
+ allow(Gitlab::Geo).to receive(:secondary?).and_return(true)
+
+ expect(remote_mirror.sync).to be_nil
+ end
+ end
+
+ context 'with remote mirroring enabled' do
+ context 'with only protected branches enabled' do
+ context 'when it did not update in the last minute' do
+ it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+ expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+ remote_mirror.sync
+ end
+ end
+
+ context 'when it did update in the last minute' do
+ it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next minute' do
+ remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+ expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::PROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+ remote_mirror.sync
+ end
+ end
+ end
+
+ context 'with only protected branches disabled' do
+ before do
+ remote_mirror.only_protected_branches = false
+ end
+
+ context 'when it did not update in the last 5 minutes' do
+ it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+ expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+ remote_mirror.sync
+ end
+ end
+
+ context 'when it did update within the last 5 minutes' do
+ it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next 5 minutes' do
+ remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+ expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::UNPROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+ remote_mirror.sync
+ end
+ end
+ end
+ end
+ end
+
+ context '#updated_since?' do
+ let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+ let(:timestamp) { Time.now - 5.minutes }
+
+ around do |example|
+ Timecop.freeze { example.run }
+ end
+
+ before do
+ remote_mirror.update_attributes(last_update_started_at: Time.now)
+ end
+
+ context 'when remote mirror does not have status failed' do
+ it 'returns true when last update started after the timestamp' do
+ expect(remote_mirror.updated_since?(timestamp)).to be true
+ end
+
+ it 'returns false when last update started before the timestamp' do
+ expect(remote_mirror.updated_since?(Time.now + 5.minutes)).to be false
+ end
+ end
+
+ context 'when remote mirror has status failed' do
+ it 'returns false when last update started after the timestamp' do
+ remote_mirror.update_attributes(update_status: 'failed')
+
+ expect(remote_mirror.updated_since?(timestamp)).to be false
+ end
+ end
+ end
+
+ context 'no project' do
+ it 'includes mirror with a project in pending_delete' do
+ mirror = create_mirror(url: 'http://cantbeblank',
+ update_status: 'finished',
+ enabled: true,
+ last_update_at: nil,
+ updated_at: 25.hours.ago)
+ project = mirror.project
+ project.pending_delete = true
+ project.save
+ mirror.reload
+
+ expect(mirror.sync).to be_nil
+ expect(mirror.valid?).to be_truthy
+ expect(mirror.update_status).to eq('finished')
+ end
+ end
+
+ def create_mirror(params)
+ project = FactoryBot.create(:project, :repository)
+ project.remote_mirrors.create!(params)
+ end
+end
diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb
index 630b9e0519f..44d9ffd258d 100644
--- a/spec/models/repository_spec.rb
+++ b/spec/models/repository_spec.rb
@@ -758,6 +758,38 @@ describe Repository do
end
end
+ describe '#async_remove_remote' do
+ before do
+ masterrev = repository.find_branch('master').dereferenced_target
+ create_remote_branch('joe', 'remote_branch', masterrev)
+ end
+
+ context 'when worker is scheduled successfully' do
+ before do
+ masterrev = repository.find_branch('master').dereferenced_target
+ create_remote_branch('remote_name', 'remote_branch', masterrev)
+
+ allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return('1234')
+ end
+
+ it 'returns job_id' do
+ expect(repository.async_remove_remote('joe')).to eq('1234')
+ end
+ end
+
+ context 'when worker does not schedule successfully' do
+ before do
+ allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return(nil)
+ end
+
+ it 'returns nil' do
+ expect(Rails.logger).to receive(:info).with("Remove remote job failed to create for #{project.id} with remote name joe.")
+
+ expect(repository.async_remove_remote('joe')).to be_nil
+ end
+ end
+ end
+
describe '#fetch_ref' do
let(:broken_repository) { create(:project, :broken_storage).repository }
diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb
index 26fdf8d4b24..35826de5814 100644
--- a/spec/services/git_push_service_spec.rb
+++ b/spec/services/git_push_service_spec.rb
@@ -14,6 +14,72 @@ describe GitPushService, services: true do
project.add_master(user)
end
+ describe 'with remote mirrors' do
+ let(:project) { create(:project, :repository, :remote_mirror) }
+
+ subject do
+ described_class.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref)
+ end
+
+ context 'when remote mirror feature is enabled' do
+ it 'fails stuck remote mirrors' do
+ allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
+ expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+ subject.execute
+ end
+
+ it 'updates remote mirrors' do
+ expect(project).to receive(:update_remote_mirrors)
+
+ subject.execute
+ end
+ end
+
+ context 'when remote mirror feature is disabled' do
+ before do
+ stub_application_setting(mirror_available: false)
+ end
+
+ context 'with remote mirrors global setting overridden' do
+ before do
+ project.remote_mirror_available_overridden = true
+ end
+
+ it 'fails stuck remote mirrors' do
+ allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
+ expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+ subject.execute
+ end
+
+ it 'updates remote mirrors' do
+ expect(project).to receive(:update_remote_mirrors)
+
+ subject.execute
+ end
+ end
+
+ context 'without remote mirrors global setting overridden' do
+ before do
+ project.remote_mirror_available_overridden = false
+ end
+
+ it 'does not fails stuck remote mirrors' do
+ expect(project).not_to receive(:mark_stuck_remote_mirrors_as_failed!)
+
+ subject.execute
+ end
+
+ it 'does not updates remote mirrors' do
+ expect(project).not_to receive(:update_remote_mirrors)
+
+ subject.execute
+ end
+ end
+ end
+ end
+
describe 'Push branches' do
subject do
execute_service(project, user, oldrev, newrev, ref)
diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb
index b2c52214f48..b63f409579e 100644
--- a/spec/services/projects/destroy_service_spec.rb
+++ b/spec/services/projects/destroy_service_spec.rb
@@ -65,6 +65,19 @@ describe Projects::DestroyService do
Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
end
+ context 'when has remote mirrors' do
+ let!(:project) do
+ create(:project, :repository, namespace: user.namespace).tap do |project|
+ project.remote_mirrors.create(url: 'http://test.com')
+ end
+ end
+ let!(:async) { true }
+
+ it 'destroys them' do
+ expect(RemoteMirror.count).to eq(0)
+ end
+ end
+
it_behaves_like 'deleting the project'
it 'invalidates personal_project_count cache' do
diff --git a/spec/services/projects/update_remote_mirror_service_spec.rb b/spec/services/projects/update_remote_mirror_service_spec.rb
new file mode 100644
index 00000000000..be09afd9f36
--- /dev/null
+++ b/spec/services/projects/update_remote_mirror_service_spec.rb
@@ -0,0 +1,355 @@
+require 'spec_helper'
+
+describe Projects::UpdateRemoteMirrorService do
+ let(:project) { create(:project, :repository) }
+ let(:remote_project) { create(:forked_project_with_submodules) }
+ let(:repository) { project.repository }
+ let(:raw_repository) { repository.raw }
+ let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true, only_protected_branches: false) }
+
+ subject { described_class.new(project, project.creator) }
+
+ describe "#execute", :skip_gitaly_mock do
+ before do
+ create_branch(repository, 'existing-branch')
+ allow(raw_repository).to receive(:remote_tags) do
+ generate_tags(repository, 'v1.0.0', 'v1.1.0')
+ end
+ allow(raw_repository).to receive(:push_remote_branches).and_return(true)
+ end
+
+ it "fetches the remote repository" do
+ expect(repository).to receive(:fetch_remote).with(remote_mirror.remote_name, no_tags: true) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ end
+
+ subject.execute(remote_mirror)
+ end
+
+ it "succeeds" do
+ allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
+
+ result = subject.execute(remote_mirror)
+
+ expect(result[:status]).to eq(:success)
+ end
+
+ describe 'Syncing branches' do
+ it "push all the branches the first time" do
+ allow(repository).to receive(:fetch_remote)
+
+ expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, local_branch_names)
+
+ subject.execute(remote_mirror)
+ end
+
+ it "does not push anything is remote is up to date" do
+ allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
+
+ expect(raw_repository).not_to receive(:push_remote_branches)
+
+ subject.execute(remote_mirror)
+ end
+
+ it "sync new branches" do
+ # call local_branch_names early so it is not called after the new branch has been created
+ current_branches = local_branch_names
+ allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, current_branches) }
+ create_branch(repository, 'my-new-branch')
+
+ expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['my-new-branch'])
+
+ subject.execute(remote_mirror)
+ end
+
+ it "sync updated branches" do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ update_branch(repository, 'existing-branch')
+ end
+
+ expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
+
+ subject.execute(remote_mirror)
+ end
+
+ context 'when push only protected branches option is set' do
+ let(:unprotected_branch_name) { 'existing-branch' }
+ let(:protected_branch_name) do
+ project.repository.branch_names.find { |n| n != unprotected_branch_name }
+ end
+ let!(:protected_branch) do
+ create(:protected_branch, project: project, name: protected_branch_name)
+ end
+
+ before do
+ project.reload
+ remote_mirror.only_protected_branches = true
+ end
+
+ it "sync updated protected branches" do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ update_branch(repository, protected_branch_name)
+ end
+
+ expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
+
+ subject.execute(remote_mirror)
+ end
+
+ it 'does not sync unprotected branches' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ update_branch(repository, unprotected_branch_name)
+ end
+
+ expect(raw_repository).not_to receive(:push_remote_branches).with(remote_mirror.remote_name, [unprotected_branch_name])
+
+ subject.execute(remote_mirror)
+ end
+ end
+
+ context 'when branch exists in local and remote repo' do
+ context 'when it has diverged' do
+ it 'syncs branches' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ update_remote_branch(repository, remote_mirror.remote_name, 'markdown')
+ end
+
+ expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['markdown'])
+
+ subject.execute(remote_mirror)
+ end
+ end
+ end
+
+ describe 'for delete' do
+ context 'when branch exists in local and remote repo' do
+ it 'deletes the branch from remote repo' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ delete_branch(repository, 'existing-branch')
+ end
+
+ expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
+
+ subject.execute(remote_mirror)
+ end
+ end
+
+ context 'when push only protected branches option is set' do
+ before do
+ remote_mirror.only_protected_branches = true
+ end
+
+ context 'when branch exists in local and remote repo' do
+ let!(:protected_branch_name) { local_branch_names.first }
+
+ before do
+ create(:protected_branch, project: project, name: protected_branch_name)
+ project.reload
+ end
+
+ it 'deletes the protected branch from remote repo' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ delete_branch(repository, protected_branch_name)
+ end
+
+ expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
+
+ subject.execute(remote_mirror)
+ end
+
+ it 'does not delete the unprotected branch from remote repo' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+ delete_branch(repository, 'existing-branch')
+ end
+
+ expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
+
+ subject.execute(remote_mirror)
+ end
+ end
+
+ context 'when branch only exists on remote repo' do
+ let!(:protected_branch_name) { 'remote-branch' }
+
+ before do
+ create(:protected_branch, project: project, name: protected_branch_name)
+ end
+
+ context 'when it has diverged' do
+ it 'does not delete the remote branch' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+ rev = repository.find_branch('markdown').dereferenced_target
+ create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id)
+ end
+
+ expect(raw_repository).not_to receive(:delete_remote_branches)
+
+ subject.execute(remote_mirror)
+ end
+ end
+
+ context 'when it has not diverged' do
+ it 'deletes the remote branch' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+ masterrev = repository.find_branch('master').dereferenced_target
+ create_remote_branch(repository, remote_mirror.remote_name, protected_branch_name, masterrev.id)
+ end
+
+ expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
+
+ subject.execute(remote_mirror)
+ end
+ end
+ end
+ end
+
+ context 'when branch only exists on remote repo' do
+ context 'when it has diverged' do
+ it 'does not delete the remote branch' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+ rev = repository.find_branch('markdown').dereferenced_target
+ create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id)
+ end
+
+ expect(raw_repository).not_to receive(:delete_remote_branches)
+
+ subject.execute(remote_mirror)
+ end
+ end
+
+ context 'when it has not diverged' do
+ it 'deletes the remote branch' do
+ allow(repository).to receive(:fetch_remote) do
+ sync_remote(repository, remote_mirror.remote_name, local_branch_names)
+
+ masterrev = repository.find_branch('master').dereferenced_target
+ create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', masterrev.id)
+ end
+
+ expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['remote-branch'])
+
+ subject.execute(remote_mirror)
+ end
+ end
+ end
+ end
+ end
+
+ describe 'Syncing tags' do
+ before do
+ allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
+ end
+
+ context 'when there are not tags to push' do
+ it 'does not try to push tags' do
+ allow(repository).to receive(:remote_tags) { {} }
+ allow(repository).to receive(:tags) { [] }
+
+ expect(repository).not_to receive(:push_tags)
+
+ subject.execute(remote_mirror)
+ end
+ end
+
+ context 'when there are some tags to push' do
+ it 'pushes tags to remote' do
+ allow(raw_repository).to receive(:remote_tags) { {} }
+
+ expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['v1.0.0', 'v1.1.0'])
+
+ subject.execute(remote_mirror)
+ end
+ end
+
+ context 'when there are some tags to delete' do
+ it 'deletes tags from remote' do
+ remote_tags = generate_tags(repository, 'v1.0.0', 'v1.1.0')
+ allow(raw_repository).to receive(:remote_tags) { remote_tags }
+
+ repository.rm_tag(create(:user), 'v1.0.0')
+
+ expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['v1.0.0'])
+
+ subject.execute(remote_mirror)
+ end
+ end
+ end
+ end
+
+ def create_branch(repository, branch_name)
+ rugged = repository.rugged
+ masterrev = repository.find_branch('master').dereferenced_target
+ parentrev = repository.commit(masterrev).parent_id
+
+ rugged.references.create("refs/heads/#{branch_name}", parentrev)
+
+ repository.expire_branches_cache
+ end
+
+ def create_remote_branch(repository, remote_name, branch_name, source_id)
+ rugged = repository.rugged
+
+ rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_id)
+ end
+
+ def sync_remote(repository, remote_name, local_branch_names)
+ rugged = repository.rugged
+
+ local_branch_names.each do |branch|
+ target = repository.find_branch(branch).try(:dereferenced_target)
+ rugged.references.create("refs/remotes/#{remote_name}/#{branch}", target.id) if target
+ end
+ end
+
+ def update_remote_branch(repository, remote_name, branch)
+ rugged = repository.rugged
+ masterrev = repository.find_branch('master').dereferenced_target.id
+
+ rugged.references.create("refs/remotes/#{remote_name}/#{branch}", masterrev, force: true)
+ repository.expire_branches_cache
+ end
+
+ def update_branch(repository, branch)
+ rugged = repository.rugged
+ masterrev = repository.find_branch('master').dereferenced_target.id
+
+ # Updated existing branch
+ rugged.references.create("refs/heads/#{branch}", masterrev, force: true)
+ repository.expire_branches_cache
+ end
+
+ def delete_branch(repository, branch)
+ rugged = repository.rugged
+
+ rugged.references.delete("refs/heads/#{branch}")
+ repository.expire_branches_cache
+ end
+
+ def generate_tags(repository, *tag_names)
+ tag_names.each_with_object([]) do |name, tags|
+ tag = repository.find_tag(name)
+ target = tag.try(:target)
+ target_commit = tag.try(:dereferenced_target)
+ tags << Gitlab::Git::Tag.new(repository.raw_repository, name, target, target_commit)
+ end
+ end
+
+ def local_branch_names
+ branch_names = repository.branches.map(&:name)
+ # we want the protected branch to be pushed first
+ branch_names.unshift(branch_names.delete('master'))
+ end
+end
diff --git a/spec/workers/repository_remove_remote_worker_spec.rb b/spec/workers/repository_remove_remote_worker_spec.rb
new file mode 100644
index 00000000000..f22d7c1d073
--- /dev/null
+++ b/spec/workers/repository_remove_remote_worker_spec.rb
@@ -0,0 +1,50 @@
+require 'rails_helper'
+
+describe RepositoryRemoveRemoteWorker do
+ subject(:worker) { described_class.new }
+
+ describe '#perform' do
+ let(:remote_name) { 'joe'}
+ let!(:project) { create(:project, :repository) }
+
+ context 'when it cannot obtain lease' do
+ it 'logs error' do
+ allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
+
+ expect_any_instance_of(Repository).not_to receive(:remove_remote)
+ expect(worker).to receive(:log_error).with('Cannot obtain an exclusive lease. There must be another instance already in execution.')
+
+ worker.perform(project.id, remote_name)
+ end
+ end
+
+ context 'when it gets the lease' do
+ before do
+ allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
+ end
+
+ context 'when project does not exist' do
+ it 'returns nil' do
+ expect(worker.perform(-1, 'remote_name')).to be_nil
+ end
+ end
+
+ context 'when project exists' do
+ it 'removes remote from repository' do
+ masterrev = project.repository.find_branch('master').dereferenced_target
+
+ create_remote_branch(remote_name, 'remote_branch', masterrev)
+
+ expect_any_instance_of(Repository).to receive(:remove_remote).with(remote_name).and_call_original
+
+ worker.perform(project.id, remote_name)
+ end
+ end
+ end
+ end
+
+ def create_remote_branch(remote_name, branch_name, target)
+ rugged = project.repository.rugged
+ rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
+ end
+end
diff --git a/spec/workers/repository_update_remote_mirror_worker_spec.rb b/spec/workers/repository_update_remote_mirror_worker_spec.rb
new file mode 100644
index 00000000000..152ba2509b9
--- /dev/null
+++ b/spec/workers/repository_update_remote_mirror_worker_spec.rb
@@ -0,0 +1,84 @@
+require 'rails_helper'
+
+describe RepositoryUpdateRemoteMirrorWorker do
+ subject { described_class.new }
+
+ let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
+ let(:scheduled_time) { Time.now - 5.minutes }
+
+ around do |example|
+ Timecop.freeze(Time.now) { example.run }
+ end
+
+ describe '#perform' do
+ context 'with status none' do
+ before do
+ remote_mirror.update_attributes(update_status: 'none')
+ end
+
+ it 'sets status as finished when update remote mirror service executes successfully' do
+ expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
+
+ expect { subject.perform(remote_mirror.id, Time.now) }.to change { remote_mirror.reload.update_status }.to('finished')
+ end
+
+ it 'sets status as failed when update remote mirror service executes with errors' do
+ error_message = 'fail!'
+
+ expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :error, message: error_message)
+ expect do
+ subject.perform(remote_mirror.id, Time.now)
+ end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError, error_message)
+
+ expect(remote_mirror.reload.update_status).to eq('failed')
+ end
+
+ it 'does nothing if last_update_started_at is higher than the time the job was scheduled in' do
+ remote_mirror.update_attributes(last_update_started_at: Time.now)
+
+ expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(true)
+ expect_any_instance_of(Projects::UpdateRemoteMirrorService).not_to receive(:execute).with(remote_mirror)
+
+ expect(subject.perform(remote_mirror.id, scheduled_time)).to be_nil
+ end
+ end
+
+ context 'with unexpected error' do
+ it 'marks mirror as failed' do
+ allow_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_raise(RuntimeError)
+
+ expect do
+ subject.perform(remote_mirror.id, Time.now)
+ end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError)
+ expect(remote_mirror.reload.update_status).to eq('failed')
+ end
+ end
+
+ context 'with another worker already running' do
+ before do
+ remote_mirror.update_attributes(update_status: 'started')
+ end
+
+ it 'raises RemoteMirrorUpdateAlreadyInProgressError' do
+ expect do
+ subject.perform(remote_mirror.id, Time.now)
+ end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateAlreadyInProgressError)
+ end
+ end
+
+ context 'with status failed' do
+ before do
+ remote_mirror.update_attributes(update_status: 'failed')
+ end
+
+ it 'sets status as finished if last_update_started_at is higher than the time the job was scheduled in' do
+ remote_mirror.update_attributes(last_update_started_at: Time.now)
+
+ expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(false)
+ expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
+
+ expect { subject.perform(remote_mirror.id, scheduled_time) }.to change { remote_mirror.reload.update_status }.to('finished')
+ end
+ end
+ end
+end