Backports every CE related change from ee-5484 to CE

14 files changed, 543 insertions, 0 deletions

diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb
new file mode 100644
index 00000000000..fb5aee2e702
--- /dev/null
+++ b/app/controllers/projects/mirrors_controller.rb
@@ -0,0 +1,62 @@
+class Projects::MirrorsController < Projects::ApplicationController
+  include RepositorySettingsRedirect
+
+  # Authorize
+  before_action :authorize_admin_mirror!
+  before_action :remote_mirror, only: [:update]
+
+  layout "project_settings"
+
+  def show
+    redirect_to_repository_settings(project)
+  end
+
+  def update
+    if project.update_attributes(mirror_params)
+      flash[:notice] = 'Mirroring settings were successfully updated.'
+    else
+      flash[:alert] = project.errors.full_messages.join(', ').html_safe
+    end
+
+    respond_to do |format|
+      format.html { redirect_to_repository_settings(project) }
+      format.json do
+        if project.errors.present?
+          render json: project.errors, status: :unprocessable_entity
+        else
+          render json: ProjectMirrorSerializer.new.represent(project)
+        end
+      end
+    end
+  end
+
+  def update_now
+    if params[:sync_remote]
+      project.update_remote_mirrors
+      flash[:notice] = "The remote repository is being updated..."
+    end
+
+    redirect_to_repository_settings(project)
+  end
+
+  private
+
+  def remote_mirror
+    @remote_mirror = project.remote_mirrors.first_or_initialize
+  end
+
+  def mirror_params_attributes
+    [
+      remote_mirrors_attributes: %i[
+        url
+        id
+        enabled
+        only_protected_branches
+      ]
+    ]
+  end
+
+  def mirror_params
+    params.require(:project).permit(mirror_params_attributes)
+  end
+end
diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb
index f17056f13e0..a3bb60bb3b2 100644
--- a/app/controllers/projects/settings/repository_controller.rb
+++ b/app/controllers/projects/settings/repository_controller.rb
@@ -2,6 +2,7 @@ module Projects
   module Settings
     class RepositoryController < Projects::ApplicationController
       before_action :authorize_admin_project!
+      before_action :remote_mirror, only: [:show]
 
       def show
         render_show
@@ -25,6 +26,7 @@ module Projects
 
         define_deploy_token
         define_protected_refs
+        remote_mirror
 
         render 'show'
       end
@@ -41,6 +43,12 @@ module Projects
         load_gon_index
       end
 
+      def remote_mirror
+        return unless project.feature_available?(:repository_mirrors)
+
+        @remote_mirror = project.remote_mirrors.first_or_initialize
+      end
+
       def access_levels_options
         {
           create_access_levels: levels_for_dropdown,
diff --git a/app/models/project.rb b/app/models/project.rb
index b12b694aabd..4021e4e85bf 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -64,6 +64,9 @@ class Project < ActiveRecord::Base
   default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
 
   add_authentication_token_field :runners_token
+
+  before_validation :mark_remote_mirrors_for_removal
+
   before_save :ensure_runners_token
 
   after_save :update_project_statistics, if: :namespace_id_changed?
@@ -241,11 +244,17 @@ class Project < ActiveRecord::Base
   has_many :project_badges, class_name: 'ProjectBadge'
   has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
+  has_many :remote_mirrors, inverse_of: :project
+
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :import_data
   accepts_nested_attributes_for :auto_devops, update_only: true
 
+  accepts_nested_attributes_for :remote_mirrors,
+                                allow_destroy: true,
+                                reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
+
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :members, to: :team, prefix: true
   delegate :add_user, :add_users, to: :team
@@ -335,6 +344,7 @@ class Project < ActiveRecord::Base
   scope :with_issues_enabled, -> { with_feature_enabled(:issues) }
   scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
   scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
+  scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
 
   scope :with_group_runners_enabled, -> do
     joins(:ci_cd_settings)
@@ -754,6 +764,37 @@ class Project < ActiveRecord::Base
     import_type == 'gitea'
   end
 
+  def has_remote_mirror?
+    remote_mirror_available? && remote_mirrors.enabled.exists?
+  end
+
+  def updating_remote_mirror?
+    remote_mirrors.enabled.started.exists?
+  end
+
+  def update_remote_mirrors
+    return unless remote_mirror_available?
+
+    remote_mirrors.enabled.each(&:sync)
+  end
+
+  def mark_stuck_remote_mirrors_as_failed!
+    remote_mirrors.stuck.update_all(
+      update_status: :failed,
+      last_error: 'The remote mirror took to long to complete.',
+      last_update_at: Time.now
+    )
+  end
+
+  def mark_remote_mirrors_for_removal
+    remote_mirrors.each(&:mark_for_delete_if_blank_url)
+  end
+
+  def remote_mirror_available?
+    remote_mirror_available_overridden ||
+      ::Gitlab::CurrentSettings.mirror_available
+  end
+
   def check_limit
     unless creator.can_create_project? || namespace.kind == 'group'
       projects_limit = creator.projects_limit
diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb
new file mode 100644
index 00000000000..b7e45875830
--- /dev/null
+++ b/app/models/remote_mirror.rb
@@ -0,0 +1,216 @@
+class RemoteMirror < ActiveRecord::Base
+  include AfterCommitQueue
+
+  PROTECTED_BACKOFF_DELAY   = 1.minute
+  UNPROTECTED_BACKOFF_DELAY = 5.minutes
+
+  attr_encrypted :credentials,
+                 key: Gitlab::Application.secrets.db_key_base,
+                 marshal: true,
+                 encode: true,
+                 mode: :per_attribute_iv_and_salt,
+                 insecure_mode: true,
+                 algorithm: 'aes-256-cbc'
+
+  default_value_for :only_protected_branches, true
+
+  belongs_to :project, inverse_of: :remote_mirrors
+
+  validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
+  validates :url, addressable_url: true, if: :url_changed?
+
+  before_save :set_new_remote_name, if: :mirror_url_changed?
+
+  after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
+  after_save :refresh_remote, if: :mirror_url_changed?
+  after_update :reset_fields, if: :mirror_url_changed?
+
+  after_commit :remove_remote, on: :destroy
+
+  scope :enabled, -> { where(enabled: true) }
+  scope :started, -> { with_update_status(:started) }
+  scope :stuck,   -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
+
+  state_machine :update_status, initial: :none do
+    event :update_start do
+      transition [:none, :finished, :failed] => :started
+    end
+
+    event :update_finish do
+      transition started: :finished
+    end
+
+    event :update_fail do
+      transition started: :failed
+    end
+
+    state :started
+    state :finished
+    state :failed
+
+    after_transition any => :started do |remote_mirror, _|
+      Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path)
+
+      remote_mirror.update(last_update_started_at: Time.now)
+    end
+
+    after_transition started: :finished do |remote_mirror, _|
+      Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path)
+
+      timestamp = Time.now
+      remote_mirror.update_attributes!(
+        last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
+      )
+    end
+
+    after_transition started: :failed do |remote_mirror, _|
+      Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path)
+
+      remote_mirror.update(last_update_at: Time.now)
+    end
+  end
+
+  def remote_name
+    super || fallback_remote_name
+  end
+
+  def update_failed?
+    update_status == 'failed'
+  end
+
+  def update_in_progress?
+    update_status == 'started'
+  end
+
+  def update_repository(options)
+    raw.update(options)
+  end
+
+  def sync
+    return unless enabled?
+    return if Gitlab::Geo.secondary?
+
+    if recently_scheduled?
+      RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
+    else
+      RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
+    end
+  end
+
+  def enabled
+    return false unless project && super
+    return false unless project.remote_mirror_available?
+    return false unless project.repository_exists?
+    return false if project.pending_delete?
+
+    true
+  end
+  alias_method :enabled?, :enabled
+
+  def updated_since?(timestamp)
+    last_update_started_at && last_update_started_at > timestamp && !update_failed?
+  end
+
+  def mark_for_delete_if_blank_url
+    mark_for_destruction if url.blank?
+  end
+
+  def mark_as_failed(error_message)
+    update_fail
+    update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
+  end
+
+  def url=(value)
+    super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
+
+    mirror_url = Gitlab::UrlSanitizer.new(value)
+    self.credentials = mirror_url.credentials
+
+    super(mirror_url.sanitized_url)
+  end
+
+  def url
+    if super
+      Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
+    end
+  rescue
+    super
+  end
+
+  def safe_url
+    return if url.nil?
+
+    result = URI.parse(url)
+    result.password = '*****' if result.password
+    result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
+    result.to_s
+  end
+
+  private
+
+  def raw
+    @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
+  end
+
+  def fallback_remote_name
+    return unless id
+
+    "remote_mirror_#{id}"
+  end
+
+  def recently_scheduled?
+    return false unless self.last_update_started_at
+
+    self.last_update_started_at >= Time.now - backoff_delay
+  end
+
+  def backoff_delay
+    if self.only_protected_branches
+      PROTECTED_BACKOFF_DELAY
+    else
+      UNPROTECTED_BACKOFF_DELAY
+    end
+  end
+
+  def reset_fields
+    update_columns(
+      last_error: nil,
+      last_update_at: nil,
+      last_successful_update_at: nil,
+      update_status: 'finished'
+    )
+  end
+
+  def set_override_remote_mirror_available
+    enabled = read_attribute(:enabled)
+
+    project.update(remote_mirror_available_overridden: enabled)
+  end
+
+  def set_new_remote_name
+    self.remote_name = "remote_mirror_#{SecureRandom.hex}"
+  end
+
+  def refresh_remote
+    return unless project
+
+    # Before adding a new remote we have to delete the data from
+    # the previous remote name
+    prev_remote_name = remote_name_was || fallback_remote_name
+    run_after_commit do
+      project.repository.async_remove_remote(prev_remote_name)
+    end
+
+    project.repository.add_remote(remote_name, url)
+  end
+
+  def remove_remote
+    return unless project # could be pending to delete so don't need to touch the git repository
+
+    project.repository.async_remove_remote(remote_name)
+  end
+
+  def mirror_url_changed?
+    url_changed? || encrypted_credentials_changed?
+  end
+end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 6831305fb93..2a0802482f5 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -861,6 +861,20 @@ class Repository
     gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
   end
 
+  def async_remove_remote(remote_name)
+    return unless remote_name
+
+    job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name)
+
+    if job_id
+      Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.")
+    else
+      Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.")
+    end
+
+    job_id
+  end
+
   def fetch_source_branch!(source_repository, source_branch, local_ref)
     raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
   end
diff --git a/app/serializers/project_mirror_entity.rb b/app/serializers/project_mirror_entity.rb
new file mode 100644
index 00000000000..8051f6e7d35
--- /dev/null
+++ b/app/serializers/project_mirror_entity.rb
@@ -0,0 +1,13 @@
+class ProjectMirrorEntity < Grape::Entity
+  prepend ::EE::ProjectMirrorEntity
+
+  expose :id
+
+  expose :remote_mirrors_attributes do |project|
+    next [] unless project.remote_mirrors.present?
+
+    project.remote_mirrors.map do |remote|
+      remote.as_json(only: %i[id url enabled])
+    end
+  end
+end
diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index c037141fcde..f3bfc53dcd3 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -55,6 +55,7 @@ class GitPushService < BaseService
     execute_related_hooks
     perform_housekeeping
 
+    update_remote_mirrors
     update_caches
 
     update_signatures
@@ -119,6 +120,13 @@ class GitPushService < BaseService
 
   protected
 
+  def update_remote_mirrors
+    return unless @project.has_remote_mirror?
+
+    @project.mark_stuck_remote_mirrors_as_failed!
+    @project.update_remote_mirrors
+  end
+
   def execute_related_hooks
     # Update merge requests that may be affected by this push. A new branch
     # could cause the last commit of a merge request to change.
diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb
new file mode 100644
index 00000000000..8183a2f26d7
--- /dev/null
+++ b/app/services/projects/update_remote_mirror_service.rb
@@ -0,0 +1,30 @@
+module Projects
+  class UpdateRemoteMirrorService < BaseService
+    attr_reader :errors
+
+    def execute(remote_mirror)
+      @errors = []
+
+      return success unless remote_mirror.enabled?
+
+      begin
+        repository.fetch_remote(remote_mirror.remote_name, no_tags: true)
+
+        opts = {}
+        if remote_mirror.only_protected_branches?
+          opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
+        end
+
+        remote_mirror.update_repository(opts)
+      rescue => e
+        errors << e.message.strip
+      end
+
+      if errors.present?
+        error(errors.join("\n\n"))
+      else
+        success
+      end
+    end
+  end
+end
diff --git a/app/views/projects/mirrors/_push.html.haml b/app/views/projects/mirrors/_push.html.haml
new file mode 100644
index 00000000000..4a6aefce351
--- /dev/null
+++ b/app/views/projects/mirrors/_push.html.haml
@@ -0,0 +1,50 @@
+- expanded = Rails.env.test?
+%section.settings.no-animate{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      Push to a remote repository
+    %button.btn.js-settings-toggle
+      = expanded ? 'Collapse' : 'Expand'
+    %p
+      Set up the remote repository that you want to update with the content of the current repository
+      every time someone pushes to it.
+      = link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
+  .settings-content
+    = form_for @project, url: project_mirror_path(@project) do |f|
+      %div
+        = form_errors(@project)
+      = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
+      - if @remote_mirror.last_error.present?
+        .panel.panel-danger
+          .panel-heading
+            - if @remote_mirror.last_update_at
+              The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
+            - else
+              The remote repository failed to update.
+
+            - if @remote_mirror.last_successful_update_at
+              Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
+          .panel-body
+            %pre
+              :preserve
+                #{h(@remote_mirror.last_error.strip)}
+      = f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
+        .form-group
+          = rm_form.check_box :enabled, class: "pull-left"
+          .prepend-left-20
+            = rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
+            %p.light.append-bottom-0
+              Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
+        .form-group.has-feedback
+          = rm_form.label :url, "Git repository URL", class: "label-light"
+          = rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
+
+          = render "projects/mirrors/instructions"
+
+        .form-group
+          = rm_form.check_box :only_protected_branches, class: 'pull-left'
+          .prepend-left-20
+            = rm_form.label :only_protected_branches, class: 'label-light'
+            = link_to icon('question-circle'), help_page_path('user/project/protected_branches')
+
+        = f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
diff --git a/app/views/projects/mirrors/_show.html.haml b/app/views/projects/mirrors/_show.html.haml
new file mode 100644
index 00000000000..da789906aef
--- /dev/null
+++ b/app/views/projects/mirrors/_show.html.haml
@@ -0,0 +1,3 @@
+- if can?(current_user, :admin_mirror, @project)
+	= render 'projects/mirrors/push'
+
diff --git a/app/views/shared/_remote_mirror_update_button.html.haml b/app/views/shared/_remote_mirror_update_button.html.haml
new file mode 100644
index 00000000000..34de1c0695f
--- /dev/null
+++ b/app/views/shared/_remote_mirror_update_button.html.haml
@@ -0,0 +1,13 @@
+- if @project.has_remote_mirror?
+  .append-bottom-default
+    - if remote_mirror.update_in_progress?
+      %span.btn.disabled
+        = icon("refresh spin")
+        Updating&hellip;
+    - else
+      = link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do
+        = icon("refresh")
+        Update Now
+    - if @remote_mirror.last_successful_update_at
+      %p.inline.prepend-left-10
+        Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index c469aea7052..d41bcd1abcb 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -112,3 +112,4 @@
 - update_user_activity
 - upload_checksum
 - web_hook
+- repository_update_remote_mirror
diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb
new file mode 100644
index 00000000000..1c19b604b77
--- /dev/null
+++ b/app/workers/repository_remove_remote_worker.rb
@@ -0,0 +1,35 @@
+class RepositoryRemoveRemoteWorker
+  include ApplicationWorker
+  include ExclusiveLeaseGuard
+
+  LEASE_TIMEOUT = 1.hour
+
+  attr_reader :project, :remote_name
+
+  def perform(project_id, remote_name)
+    @remote_name = remote_name
+    @project = Project.find_by_id(project_id)
+
+    return unless @project
+
+    logger.info("Removing remote #{remote_name} from project #{project.id}")
+
+    try_obtain_lease do
+      remove_remote = @project.repository.remove_remote(remote_name)
+
+      if remove_remote
+        logger.info("Remote #{remote_name} was successfully removed from project #{project.id}")
+      else
+        logger.error("Could not remove remote #{remote_name} from project #{project.id}")
+      end
+    end
+  end
+
+  def lease_timeout
+    LEASE_TIMEOUT
+  end
+
+  def lease_key
+    "remove_remote_#{project.id}_#{remote_name}"
+  end
+end
diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb
new file mode 100644
index 00000000000..bb963979e88
--- /dev/null
+++ b/app/workers/repository_update_remote_mirror_worker.rb
@@ -0,0 +1,49 @@
+class RepositoryUpdateRemoteMirrorWorker
+  UpdateAlreadyInProgressError = Class.new(StandardError)
+  UpdateError = Class.new(StandardError)
+
+  include ApplicationWorker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options retry: 3, dead: false
+
+  sidekiq_retry_in { |count| 30 * count }
+
+  sidekiq_retries_exhausted do |msg, _|
+    Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
+  end
+
+  def perform(remote_mirror_id, scheduled_time)
+    remote_mirror = RemoteMirror.find(remote_mirror_id)
+    return if remote_mirror.updated_since?(scheduled_time)
+
+    raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
+
+    remote_mirror.update_start
+
+    project = remote_mirror.project
+    current_user = project.creator
+    result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
+    raise UpdateError, result[:message] if result[:status] == :error
+
+    remote_mirror.update_finish
+  rescue UpdateAlreadyInProgressError
+    raise
+  rescue UpdateError => ex
+    fail_remote_mirror(remote_mirror, ex.message)
+    raise
+  rescue => ex
+    return unless remote_mirror
+
+    fail_remote_mirror(remote_mirror, ex.message)
+    raise UpdateError, "#{ex.class}: #{ex.message}"
+  end
+
+  private
+
+  def fail_remote_mirror(remote_mirror, message)
+    remote_mirror.mark_as_failed(message)
+
+    Rails.logger.error(message)
+  end
+end