diff options
Diffstat (limited to 'app')
-rw-r--r-- | app/controllers/projects/mirrors_controller.rb | 62 | ||||
-rw-r--r-- | app/controllers/projects/settings/repository_controller.rb | 8 | ||||
-rw-r--r-- | app/models/project.rb | 41 | ||||
-rw-r--r-- | app/models/remote_mirror.rb | 216 | ||||
-rw-r--r-- | app/models/repository.rb | 14 | ||||
-rw-r--r-- | app/serializers/project_mirror_entity.rb | 13 | ||||
-rw-r--r-- | app/services/git_push_service.rb | 8 | ||||
-rw-r--r-- | app/services/projects/update_remote_mirror_service.rb | 30 | ||||
-rw-r--r-- | app/views/projects/mirrors/_push.html.haml | 50 | ||||
-rw-r--r-- | app/views/projects/mirrors/_show.html.haml | 3 | ||||
-rw-r--r-- | app/views/shared/_remote_mirror_update_button.html.haml | 13 | ||||
-rw-r--r-- | app/workers/all_queues.yml | 1 | ||||
-rw-r--r-- | app/workers/repository_remove_remote_worker.rb | 35 | ||||
-rw-r--r-- | app/workers/repository_update_remote_mirror_worker.rb | 49 |
14 files changed, 543 insertions, 0 deletions
diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb new file mode 100644 index 00000000000..fb5aee2e702 --- /dev/null +++ b/app/controllers/projects/mirrors_controller.rb @@ -0,0 +1,62 @@ +class Projects::MirrorsController < Projects::ApplicationController + include RepositorySettingsRedirect + + # Authorize + before_action :authorize_admin_mirror! + before_action :remote_mirror, only: [:update] + + layout "project_settings" + + def show + redirect_to_repository_settings(project) + end + + def update + if project.update_attributes(mirror_params) + flash[:notice] = 'Mirroring settings were successfully updated.' + else + flash[:alert] = project.errors.full_messages.join(', ').html_safe + end + + respond_to do |format| + format.html { redirect_to_repository_settings(project) } + format.json do + if project.errors.present? + render json: project.errors, status: :unprocessable_entity + else + render json: ProjectMirrorSerializer.new.represent(project) + end + end + end + end + + def update_now + if params[:sync_remote] + project.update_remote_mirrors + flash[:notice] = "The remote repository is being updated..." + end + + redirect_to_repository_settings(project) + end + + private + + def remote_mirror + @remote_mirror = project.remote_mirrors.first_or_initialize + end + + def mirror_params_attributes + [ + remote_mirrors_attributes: %i[ + url + id + enabled + only_protected_branches + ] + ] + end + + def mirror_params + params.require(:project).permit(mirror_params_attributes) + end +end diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb index f17056f13e0..a3bb60bb3b2 100644 --- a/app/controllers/projects/settings/repository_controller.rb +++ b/app/controllers/projects/settings/repository_controller.rb @@ -2,6 +2,7 @@ module Projects module Settings class RepositoryController < Projects::ApplicationController before_action :authorize_admin_project! + before_action :remote_mirror, only: [:show] def show render_show @@ -25,6 +26,7 @@ module Projects define_deploy_token define_protected_refs + remote_mirror render 'show' end @@ -41,6 +43,12 @@ module Projects load_gon_index end + def remote_mirror + return unless project.feature_available?(:repository_mirrors) + + @remote_mirror = project.remote_mirrors.first_or_initialize + end + def access_levels_options { create_access_levels: levels_for_dropdown, diff --git a/app/models/project.rb b/app/models/project.rb index b12b694aabd..4021e4e85bf 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -64,6 +64,9 @@ class Project < ActiveRecord::Base default_value_for :only_allow_merge_if_all_discussions_are_resolved, false add_authentication_token_field :runners_token + + before_validation :mark_remote_mirrors_for_removal + before_save :ensure_runners_token after_save :update_project_statistics, if: :namespace_id_changed? @@ -241,11 +244,17 @@ class Project < ActiveRecord::Base has_many :project_badges, class_name: 'ProjectBadge' has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :remote_mirrors, inverse_of: :project + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data accepts_nested_attributes_for :auto_devops, update_only: true + accepts_nested_attributes_for :remote_mirrors, + allow_destroy: true, + reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? } + delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team @@ -335,6 +344,7 @@ class Project < ActiveRecord::Base scope :with_issues_enabled, -> { with_feature_enabled(:issues) } scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) } scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } + scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } scope :with_group_runners_enabled, -> do joins(:ci_cd_settings) @@ -754,6 +764,37 @@ class Project < ActiveRecord::Base import_type == 'gitea' end + def has_remote_mirror? + remote_mirror_available? && remote_mirrors.enabled.exists? + end + + def updating_remote_mirror? + remote_mirrors.enabled.started.exists? + end + + def update_remote_mirrors + return unless remote_mirror_available? + + remote_mirrors.enabled.each(&:sync) + end + + def mark_stuck_remote_mirrors_as_failed! + remote_mirrors.stuck.update_all( + update_status: :failed, + last_error: 'The remote mirror took to long to complete.', + last_update_at: Time.now + ) + end + + def mark_remote_mirrors_for_removal + remote_mirrors.each(&:mark_for_delete_if_blank_url) + end + + def remote_mirror_available? + remote_mirror_available_overridden || + ::Gitlab::CurrentSettings.mirror_available + end + def check_limit unless creator.can_create_project? || namespace.kind == 'group' projects_limit = creator.projects_limit diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb new file mode 100644 index 00000000000..b7e45875830 --- /dev/null +++ b/app/models/remote_mirror.rb @@ -0,0 +1,216 @@ +class RemoteMirror < ActiveRecord::Base + include AfterCommitQueue + + PROTECTED_BACKOFF_DELAY = 1.minute + UNPROTECTED_BACKOFF_DELAY = 5.minutes + + attr_encrypted :credentials, + key: Gitlab::Application.secrets.db_key_base, + marshal: true, + encode: true, + mode: :per_attribute_iv_and_salt, + insecure_mode: true, + algorithm: 'aes-256-cbc' + + default_value_for :only_protected_branches, true + + belongs_to :project, inverse_of: :remote_mirrors + + validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true } + validates :url, addressable_url: true, if: :url_changed? + + before_save :set_new_remote_name, if: :mirror_url_changed? + + after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available } + after_save :refresh_remote, if: :mirror_url_changed? + after_update :reset_fields, if: :mirror_url_changed? + + after_commit :remove_remote, on: :destroy + + scope :enabled, -> { where(enabled: true) } + scope :started, -> { with_update_status(:started) } + scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + + state_machine :update_status, initial: :none do + event :update_start do + transition [:none, :finished, :failed] => :started + end + + event :update_finish do + transition started: :finished + end + + event :update_fail do + transition started: :failed + end + + state :started + state :finished + state :failed + + after_transition any => :started do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_started_at: Time.now) + end + + after_transition started: :finished do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path) + + timestamp = Time.now + remote_mirror.update_attributes!( + last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil + ) + end + + after_transition started: :failed do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_at: Time.now) + end + end + + def remote_name + super || fallback_remote_name + end + + def update_failed? + update_status == 'failed' + end + + def update_in_progress? + update_status == 'started' + end + + def update_repository(options) + raw.update(options) + end + + def sync + return unless enabled? + return if Gitlab::Geo.secondary? + + if recently_scheduled? + RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) + else + RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now) + end + end + + def enabled + return false unless project && super + return false unless project.remote_mirror_available? + return false unless project.repository_exists? + return false if project.pending_delete? + + true + end + alias_method :enabled?, :enabled + + def updated_since?(timestamp) + last_update_started_at && last_update_started_at > timestamp && !update_failed? + end + + def mark_for_delete_if_blank_url + mark_for_destruction if url.blank? + end + + def mark_as_failed(error_message) + update_fail + update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message)) + end + + def url=(value) + super(value) && return unless Gitlab::UrlSanitizer.valid?(value) + + mirror_url = Gitlab::UrlSanitizer.new(value) + self.credentials = mirror_url.credentials + + super(mirror_url.sanitized_url) + end + + def url + if super + Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url + end + rescue + super + end + + def safe_url + return if url.nil? + + result = URI.parse(url) + result.password = '*****' if result.password + result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.to_s + end + + private + + def raw + @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name) + end + + def fallback_remote_name + return unless id + + "remote_mirror_#{id}" + end + + def recently_scheduled? + return false unless self.last_update_started_at + + self.last_update_started_at >= Time.now - backoff_delay + end + + def backoff_delay + if self.only_protected_branches + PROTECTED_BACKOFF_DELAY + else + UNPROTECTED_BACKOFF_DELAY + end + end + + def reset_fields + update_columns( + last_error: nil, + last_update_at: nil, + last_successful_update_at: nil, + update_status: 'finished' + ) + end + + def set_override_remote_mirror_available + enabled = read_attribute(:enabled) + + project.update(remote_mirror_available_overridden: enabled) + end + + def set_new_remote_name + self.remote_name = "remote_mirror_#{SecureRandom.hex}" + end + + def refresh_remote + return unless project + + # Before adding a new remote we have to delete the data from + # the previous remote name + prev_remote_name = remote_name_was || fallback_remote_name + run_after_commit do + project.repository.async_remove_remote(prev_remote_name) + end + + project.repository.add_remote(remote_name, url) + end + + def remove_remote + return unless project # could be pending to delete so don't need to touch the git repository + + project.repository.async_remove_remote(remote_name) + end + + def mirror_url_changed? + url_changed? || encrypted_credentials_changed? + end +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 6831305fb93..2a0802482f5 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -861,6 +861,20 @@ class Repository gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end + def async_remove_remote(remote_name) + return unless remote_name + + job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name) + + if job_id + Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.") + else + Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.") + end + + job_id + end + def fetch_source_branch!(source_repository, source_branch, local_ref) raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) end diff --git a/app/serializers/project_mirror_entity.rb b/app/serializers/project_mirror_entity.rb new file mode 100644 index 00000000000..8051f6e7d35 --- /dev/null +++ b/app/serializers/project_mirror_entity.rb @@ -0,0 +1,13 @@ +class ProjectMirrorEntity < Grape::Entity + prepend ::EE::ProjectMirrorEntity + + expose :id + + expose :remote_mirrors_attributes do |project| + next [] unless project.remote_mirrors.present? + + project.remote_mirrors.map do |remote| + remote.as_json(only: %i[id url enabled]) + end + end +end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index c037141fcde..f3bfc53dcd3 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -55,6 +55,7 @@ class GitPushService < BaseService execute_related_hooks perform_housekeeping + update_remote_mirrors update_caches update_signatures @@ -119,6 +120,13 @@ class GitPushService < BaseService protected + def update_remote_mirrors + return unless @project.has_remote_mirror? + + @project.mark_stuck_remote_mirrors_as_failed! + @project.update_remote_mirrors + end + def execute_related_hooks # Update merge requests that may be affected by this push. A new branch # could cause the last commit of a merge request to change. diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb new file mode 100644 index 00000000000..8183a2f26d7 --- /dev/null +++ b/app/services/projects/update_remote_mirror_service.rb @@ -0,0 +1,30 @@ +module Projects + class UpdateRemoteMirrorService < BaseService + attr_reader :errors + + def execute(remote_mirror) + @errors = [] + + return success unless remote_mirror.enabled? + + begin + repository.fetch_remote(remote_mirror.remote_name, no_tags: true) + + opts = {} + if remote_mirror.only_protected_branches? + opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) + end + + remote_mirror.update_repository(opts) + rescue => e + errors << e.message.strip + end + + if errors.present? + error(errors.join("\n\n")) + else + success + end + end + end +end diff --git a/app/views/projects/mirrors/_push.html.haml b/app/views/projects/mirrors/_push.html.haml new file mode 100644 index 00000000000..4a6aefce351 --- /dev/null +++ b/app/views/projects/mirrors/_push.html.haml @@ -0,0 +1,50 @@ +- expanded = Rails.env.test? +%section.settings.no-animate{ class: ('expanded' if expanded) } + .settings-header + %h4 + Push to a remote repository + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + Set up the remote repository that you want to update with the content of the current repository + every time someone pushes to it. + = link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank' + .settings-content + = form_for @project, url: project_mirror_path(@project) do |f| + %div + = form_errors(@project) + = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror + - if @remote_mirror.last_error.present? + .panel.panel-danger + .panel-heading + - if @remote_mirror.last_update_at + The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}. + - else + The remote repository failed to update. + + - if @remote_mirror.last_successful_update_at + Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. + .panel-body + %pre + :preserve + #{h(@remote_mirror.last_error.strip)} + = f.fields_for :remote_mirrors, @remote_mirror do |rm_form| + .form-group + = rm_form.check_box :enabled, class: "pull-left" + .prepend-left-20 + = rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0" + %p.light.append-bottom-0 + Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it. + .form-group.has-feedback + = rm_form.label :url, "Git repository URL", class: "label-light" + = rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git' + + = render "projects/mirrors/instructions" + + .form-group + = rm_form.check_box :only_protected_branches, class: 'pull-left' + .prepend-left-20 + = rm_form.label :only_protected_branches, class: 'label-light' + = link_to icon('question-circle'), help_page_path('user/project/protected_branches') + + = f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror' diff --git a/app/views/projects/mirrors/_show.html.haml b/app/views/projects/mirrors/_show.html.haml new file mode 100644 index 00000000000..da789906aef --- /dev/null +++ b/app/views/projects/mirrors/_show.html.haml @@ -0,0 +1,3 @@ +- if can?(current_user, :admin_mirror, @project) + = render 'projects/mirrors/push' + diff --git a/app/views/shared/_remote_mirror_update_button.html.haml b/app/views/shared/_remote_mirror_update_button.html.haml new file mode 100644 index 00000000000..34de1c0695f --- /dev/null +++ b/app/views/shared/_remote_mirror_update_button.html.haml @@ -0,0 +1,13 @@ +- if @project.has_remote_mirror? + .append-bottom-default + - if remote_mirror.update_in_progress? + %span.btn.disabled + = icon("refresh spin") + Updating… + - else + = link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do + = icon("refresh") + Update Now + - if @remote_mirror.last_successful_update_at + %p.inline.prepend-left-10 + Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index c469aea7052..d41bcd1abcb 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -112,3 +112,4 @@ - update_user_activity - upload_checksum - web_hook +- repository_update_remote_mirror diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb new file mode 100644 index 00000000000..1c19b604b77 --- /dev/null +++ b/app/workers/repository_remove_remote_worker.rb @@ -0,0 +1,35 @@ +class RepositoryRemoveRemoteWorker + include ApplicationWorker + include ExclusiveLeaseGuard + + LEASE_TIMEOUT = 1.hour + + attr_reader :project, :remote_name + + def perform(project_id, remote_name) + @remote_name = remote_name + @project = Project.find_by_id(project_id) + + return unless @project + + logger.info("Removing remote #{remote_name} from project #{project.id}") + + try_obtain_lease do + remove_remote = @project.repository.remove_remote(remote_name) + + if remove_remote + logger.info("Remote #{remote_name} was successfully removed from project #{project.id}") + else + logger.error("Could not remove remote #{remote_name} from project #{project.id}") + end + end + end + + def lease_timeout + LEASE_TIMEOUT + end + + def lease_key + "remove_remote_#{project.id}_#{remote_name}" + end +end diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb new file mode 100644 index 00000000000..bb963979e88 --- /dev/null +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -0,0 +1,49 @@ +class RepositoryUpdateRemoteMirrorWorker + UpdateAlreadyInProgressError = Class.new(StandardError) + UpdateError = Class.new(StandardError) + + include ApplicationWorker + include Gitlab::ShellAdapter + + sidekiq_options retry: 3, dead: false + + sidekiq_retry_in { |count| 30 * count } + + sidekiq_retries_exhausted do |msg, _| + Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}" + end + + def perform(remote_mirror_id, scheduled_time) + remote_mirror = RemoteMirror.find(remote_mirror_id) + return if remote_mirror.updated_since?(scheduled_time) + + raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress? + + remote_mirror.update_start + + project = remote_mirror.project + current_user = project.creator + result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror) + raise UpdateError, result[:message] if result[:status] == :error + + remote_mirror.update_finish + rescue UpdateAlreadyInProgressError + raise + rescue UpdateError => ex + fail_remote_mirror(remote_mirror, ex.message) + raise + rescue => ex + return unless remote_mirror + + fail_remote_mirror(remote_mirror, ex.message) + raise UpdateError, "#{ex.class}: #{ex.message}" + end + + private + + def fail_remote_mirror(remote_mirror, message) + remote_mirror.mark_as_failed(message) + + Rails.logger.error(message) + end +end |