Adds remote mirror table migration

16 files changed, 118 insertions, 13 deletions

diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb
index fb5aee2e702..5698ff4e706 100644
--- a/app/controllers/projects/mirrors_controller.rb
+++ b/app/controllers/projects/mirrors_controller.rb
@@ -2,8 +2,9 @@ class Projects::MirrorsController < Projects::ApplicationController
   include RepositorySettingsRedirect
 
   # Authorize
-  before_action :authorize_admin_mirror!
   before_action :remote_mirror, only: [:update]
+  before_action :check_mirror_available!
+  before_action :authorize_admin_project!
 
   layout "project_settings"
 
@@ -45,6 +46,10 @@ class Projects::MirrorsController < Projects::ApplicationController
     @remote_mirror = project.remote_mirrors.first_or_initialize
   end
 
+  def check_mirror_available!
+    Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin?
+  end
+
   def mirror_params_attributes
     [
       remote_mirrors_attributes: %i[
diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb
index a3bb60bb3b2..4697af4f26a 100644
--- a/app/controllers/projects/settings/repository_controller.rb
+++ b/app/controllers/projects/settings/repository_controller.rb
@@ -44,8 +44,6 @@ module Projects
       end
 
       def remote_mirror
-        return unless project.feature_available?(:repository_mirrors)
-
         @remote_mirror = project.remote_mirrors.first_or_initialize
       end
 
diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb
index 1bf98d550b0..b948e431882 100644
--- a/app/helpers/application_settings_helper.rb
+++ b/app/helpers/application_settings_helper.rb
@@ -250,7 +250,8 @@ module ApplicationSettingsHelper
       :version_check_enabled,
       :allow_local_requests_from_hooks_and_services,
       :enforce_terms,
-      :terms
+      :terms,
+      :mirror_available
     ]
   end
 end
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index a734cc7a26b..451e512aef7 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base
       gitaly_timeout_fast: 10,
       gitaly_timeout_medium: 30,
       gitaly_timeout_default: 55,
-      allow_local_requests_from_hooks_and_services: false
+      allow_local_requests_from_hooks_and_services: false,
+      mirror_available: true
     }
   end
 
diff --git a/app/models/project.rb b/app/models/project.rb
index 4021e4e85bf..3ad0d4f0c4c 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -65,7 +65,7 @@ class Project < ActiveRecord::Base
 
   add_authentication_token_field :runners_token
 
-  before_validation :mark_remote_mirrors_for_removal
+  before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) }
 
   before_save :ensure_runners_token
 
diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb
index b7e45875830..3b9fd6d710f 100644
--- a/app/models/remote_mirror.rb
+++ b/app/models/remote_mirror.rb
@@ -86,9 +86,12 @@ class RemoteMirror < ActiveRecord::Base
     raw.update(options)
   end
 
+  def sync?
+    !enabled?
+  end
+
   def sync
-    return unless enabled?
-    return if Gitlab::Geo.secondary?
+    return if sync?
 
     if recently_scheduled?
       RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 2a0802482f5..b75c4aca982 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -854,7 +854,7 @@ class Repository
     add_remote(remote_name, url, mirror_refmap: refmap)
     fetch_remote(remote_name, forced: forced, prune: prune)
   ensure
-    remove_remote(remote_name) if tmp_remote_name
+    async_remove_remote(remote_name) if tmp_remote_name
   end
 
   def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true)
diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb
index 3529d0aa60c..5759b1a376f 100644
--- a/app/policies/project_policy.rb
+++ b/app/policies/project_policy.rb
@@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy
     project.merge_requests_allowing_push_to_user(user).any?
   end
 
+  with_scope :global
+  condition(:mirror_available, score: 0) do
+    ::Gitlab::CurrentSettings.current_application_settings.mirror_available
+  end
+
   # We aren't checking `:read_issue` or `:read_merge_request` in this case
   # because it could be possible for a user to see an issuable-iid
   # (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be
@@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy
     enable :create_cluster
   end
 
+  rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
+
   rule { archived }.policy do
     prevent :push_code
     prevent :push_to_delete_protected_branch
diff --git a/app/serializers/project_mirror_entity.rb b/app/serializers/project_mirror_entity.rb
index 8051f6e7d35..a9c08ac021a 100644
--- a/app/serializers/project_mirror_entity.rb
+++ b/app/serializers/project_mirror_entity.rb
@@ -1,6 +1,4 @@
 class ProjectMirrorEntity < Grape::Entity
-  prepend ::EE::ProjectMirrorEntity
-
   expose :id
 
   expose :remote_mirrors_attributes do |project|
diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb
new file mode 100644
index 00000000000..30be6accc32
--- /dev/null
+++ b/app/services/concerns/exclusive_lease_guard.rb
@@ -0,0 +1,52 @@
+#
+# Concern that helps with getting an exclusive lease for running a block
+# of code.
+#
+# `#try_obtain_lease` takes a block which will be run if it was able to
+# obtain  the lease. Implement `#lease_timeout` to configure the timeout
+# for the exclusive lease. Optionally override `#lease_key` to set the
+# lease key, it defaults to the class name with underscores.
+#
+module ExclusiveLeaseGuard
+  extend ActiveSupport::Concern
+
+  def try_obtain_lease
+    lease = exclusive_lease.try_obtain
+
+    unless lease
+      log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.')
+      return
+    end
+
+    begin
+      yield lease
+    ensure
+      release_lease(lease)
+    end
+  end
+
+  def exclusive_lease
+    @lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout)
+  end
+
+  def lease_key
+    @lease_key ||= self.class.name.underscore
+  end
+
+  def lease_timeout
+    raise NotImplementedError,
+          "#{self.class.name} does not implement #{__method__}"
+  end
+
+  def release_lease(uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+  end
+
+  def renew_lease!
+    exclusive_lease.renew
+  end
+
+  def log_error(message, extra_args = {})
+    logger.error(message)
+  end
+end
diff --git a/app/views/admin/application_settings/_repository_mirrors_form.html.haml b/app/views/admin/application_settings/_repository_mirrors_form.html.haml
new file mode 100644
index 00000000000..09183ec6260
--- /dev/null
+++ b/app/views/admin/application_settings/_repository_mirrors_form.html.haml
@@ -0,0 +1,16 @@
+= form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f|
+  = form_errors(@application_setting)
+
+  %fieldset
+    .form-group
+      = f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2'
+      .col-sm-10
+        .checkbox
+          = f.label :mirror_available do
+            = f.check_box :mirror_available
+            Allow mirrors to be setup for projects
+        %span.help-block
+          If disabled, only admins will be able to setup mirrors in projects.
+          = link_to icon('question-circle'), help_page_path('workflow/repository_mirroring')
+
+  = f.submit 'Save changes', class: "btn btn-success"
diff --git a/app/views/admin/application_settings/show.html.haml b/app/views/admin/application_settings/show.html.haml
index 3c00e3c8fc4..4d686d02046 100644
--- a/app/views/admin/application_settings/show.html.haml
+++ b/app/views/admin/application_settings/show.html.haml
@@ -313,3 +313,14 @@
       = _('Allow requests to the local network from hooks and services.')
   .settings-content
     = render 'outbound'
+
+%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      = _('Repository mirror settings')
+    %button.btn.js-settings-toggle{ type: 'button' }
+      = expanded ? 'Collapse' : 'Expand'
+    %p
+      = _('Configure push and pull mirrors.')
+  .settings-content
+    = render partial: 'repository_mirrors_form'
diff --git a/app/views/projects/mirrors/_instructions.html.haml b/app/views/projects/mirrors/_instructions.html.haml
new file mode 100644
index 00000000000..64f0fde30cf
--- /dev/null
+++ b/app/views/projects/mirrors/_instructions.html.haml
@@ -0,0 +1,10 @@
+.account-well.prepend-top-default.append-bottom-default
+  %ul
+    %li
+      The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>.
+    %li
+      Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>.
+    %li
+      The update action will time out after 10 minutes. For big repositories, use a clone/push combination.
+    %li
+      The Git LFS objects will <strong>not</strong> be synced.
diff --git a/app/views/projects/mirrors/_show.html.haml b/app/views/projects/mirrors/_show.html.haml
index da789906aef..de77701a373 100644
--- a/app/views/projects/mirrors/_show.html.haml
+++ b/app/views/projects/mirrors/_show.html.haml
@@ -1,3 +1,3 @@
-- if can?(current_user, :admin_mirror, @project)
-	= render 'projects/mirrors/push'
+- if can?(current_user, :admin_remote_mirror, @project)
+  = render 'projects/mirrors/push'
 
diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml
index f57590a908f..5dda2ec28b4 100644
--- a/app/views/projects/settings/repository/show.html.haml
+++ b/app/views/projects/settings/repository/show.html.haml
@@ -2,6 +2,8 @@
 - page_title "Repository"
 - @content_class = "limit-container-width" unless fluid_layout
 
+= render "projects/mirrors/show"
+
 -# Protected branches & tags use a lot of nested partials.
 -# The shared parts of the views can be found in the `shared` directory.
 -# Those are used throughout the actual views. These `shared` views are then
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index d41bcd1abcb..1949522d6f0 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -106,6 +106,7 @@
 - rebase
 - repository_fork
 - repository_import
+- repository_remove_remote
 - storage_migrator
 - system_hook_push
 - update_merge_requests