From ab11eee1d6e3881399b671f0ebe857a085321371 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 19 Jul 2019 17:28:23 +0200 Subject: Mark mirrors as failed 1 hour after they started We call `Project#mark_stuck_remote_mirrors_as_failed!` from the `Git::BaseHooksService`. So that gets called every time we push tags or branches. Before this would only mark started mirrors as failed if they had been started 24 hours ago. A push would never take 24 hours, especially not when we run it so often. Lowering that threshold 1 hour should at least allow us to retry broken mirrors more often on pushes. The timeout for the initial push is set somewhat longer to accommodate for pushing large repos. Both numbers are currently picked arbitrarily. --- app/models/remote_mirror.rb | 4 ++-- .../bvl-mark-remote-mirrors-as-failed-sooner.yml | 5 +++++ spec/models/remote_mirror_spec.rb | 15 ++++++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 changelogs/unreleased/bvl-mark-remote-mirrors-as-failed-sooner.yml diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index af705b29f7a..6b5605f9999 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -31,7 +31,7 @@ class RemoteMirror < ApplicationRecord scope :enabled, -> { where(enabled: true) } scope :started, -> { with_update_status(:started) } - scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.hour.ago, 3.hours.ago) } state_machine :update_status, initial: :none do event :update_start do @@ -173,7 +173,7 @@ class RemoteMirror < ApplicationRecord result = URI.parse(url) result.password = '*****' if result.password - result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.user = '*****' if result.user && result.user != 'git' # tokens or other data may be saved as user result.to_s end diff --git a/changelogs/unreleased/bvl-mark-remote-mirrors-as-failed-sooner.yml b/changelogs/unreleased/bvl-mark-remote-mirrors-as-failed-sooner.yml new file mode 100644 index 00000000000..1db0a4952b2 --- /dev/null +++ b/changelogs/unreleased/bvl-mark-remote-mirrors-as-failed-sooner.yml @@ -0,0 +1,5 @@ +--- +title: Mark push mirrors as failed after 1 hour +merge_request: 30999 +author: +type: changed diff --git a/spec/models/remote_mirror_spec.rb b/spec/models/remote_mirror_spec.rb index e14b19db915..687b0935c55 100644 --- a/spec/models/remote_mirror_spec.rb +++ b/spec/models/remote_mirror_spec.rb @@ -113,7 +113,7 @@ describe RemoteMirror, :mailer do remote_mirror = create(:remote_mirror) - expect(remote_mirror.remote_name).to eq("remote_mirror_secret") + expect(remote_mirror.remote_name).to eq('remote_mirror_secret') end end @@ -201,11 +201,20 @@ describe RemoteMirror, :mailer do end context 'stuck mirrors' do - it 'includes mirrors stuck in started with no last_update_at set' do + it 'includes mirrors that were started over an hour ago' do + mirror = create_mirror(url: 'http://cantbeblank', + update_status: 'started', + last_update_at: 3.hours.ago, + updated_at: 2.hours.ago) + + expect(described_class.stuck.last).to eq(mirror) + end + + it 'includes mirrors started over 3 hours ago for their first sync' do mirror = create_mirror(url: 'http://cantbeblank', update_status: 'started', last_update_at: nil, - updated_at: 25.hours.ago) + updated_at: 4.hours.ago) expect(described_class.stuck.last).to eq(mirror) end -- cgit v1.2.1