Mark mirrors as failed 1 hour after they started

We call `Project#mark_stuck_remote_mirrors_as_failed!` from the
`Git::BaseHooksService`. So that gets called every time we push tags
or branches.

Before this would only mark started mirrors as failed if they had
been started 24 hours ago. A push would never take 24 hours,
especially not when we run it so often.

Lowering that threshold 1 hour should at least allow us to retry
broken mirrors more often on pushes.

The timeout for the initial push is set somewhat longer to accommodate
for pushing large repos. Both numbers are currently picked arbitrarily.

3 files changed, 19 insertions, 5 deletions

diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb
index af705b29f7a..6b5605f9999 100644
--- a/app/models/remote_mirror.rb
+++ b/app/models/remote_mirror.rb
@@ -31,7 +31,7 @@ class RemoteMirror < ApplicationRecord
 
   scope :enabled, -> { where(enabled: true) }
   scope :started, -> { with_update_status(:started) }
-  scope :stuck,   -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
+  scope :stuck,   -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.hour.ago, 3.hours.ago) }
 
   state_machine :update_status, initial: :none do
     event :update_start do
@@ -173,7 +173,7 @@ class RemoteMirror < ApplicationRecord
 
     result = URI.parse(url)
     result.password = '*****' if result.password
-    result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
+    result.user = '*****' if result.user && result.user != 'git' # tokens or other data may be saved as user
     result.to_s
   end
 
diff --git a/changelogs/unreleased/bvl-mark-remote-mirrors-as-failed-sooner.yml b/changelogs/unreleased/bvl-mark-remote-mirrors-as-failed-sooner.yml
new file mode 100644
index 00000000000..1db0a4952b2
--- /dev/null
+++ b/changelogs/unreleased/bvl-mark-remote-mirrors-as-failed-sooner.yml
@@ -0,0 +1,5 @@
+---
+title: Mark push mirrors as failed after 1 hour
+merge_request: 30999
+author:
+type: changed
diff --git a/spec/models/remote_mirror_spec.rb b/spec/models/remote_mirror_spec.rb
index e14b19db915..687b0935c55 100644
--- a/spec/models/remote_mirror_spec.rb
+++ b/spec/models/remote_mirror_spec.rb
@@ -113,7 +113,7 @@ describe RemoteMirror, :mailer do
 
         remote_mirror = create(:remote_mirror)
 
-        expect(remote_mirror.remote_name).to eq("remote_mirror_secret")
+        expect(remote_mirror.remote_name).to eq('remote_mirror_secret')
       end
     end
 
@@ -201,11 +201,20 @@ describe RemoteMirror, :mailer do
   end
 
   context 'stuck mirrors' do
-    it 'includes mirrors stuck in started with no last_update_at set' do
+    it 'includes mirrors that were started over an hour ago' do
+      mirror = create_mirror(url: 'http://cantbeblank',
+                             update_status: 'started',
+                             last_update_at: 3.hours.ago,
+                             updated_at: 2.hours.ago)
+
+      expect(described_class.stuck.last).to eq(mirror)
+    end
+
+    it 'includes mirrors started over 3 hours ago for their first sync' do
       mirror = create_mirror(url: 'http://cantbeblank',
                              update_status: 'started',
                              last_update_at: nil,
-                             updated_at: 25.hours.ago)
+                             updated_at: 4.hours.ago)
 
       expect(described_class.stuck.last).to eq(mirror)
     end