From f7f7c3016252a09acd68c822c8950848e313ce8a Mon Sep 17 00:00:00 2001
From: Ben Kochie <bjk@gitlab.com>
Date: Wed, 17 Jul 2019 14:50:37 +0200
Subject: Update usage ping cron behavior

* Splay the start by sleeping up to one minute.
* Adjust sideikiq retries to try up to approximately three hours.
* Randomize the minute fully within the hour.
* Psudo-randomize the day of the week based on the instance UUID.
---
 app/workers/gitlab_usage_ping_worker.rb  |  6 ++++++
 changelogs/unreleased/bjk-usage_ping.yml |  5 +++++
 config/settings.rb                       | 14 +++++++++-----
 3 files changed, 20 insertions(+), 5 deletions(-)
 create mode 100644 changelogs/unreleased/bjk-usage_ping.yml

diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb
index b75e724ca98..a5e22f88a3b 100644
--- a/app/workers/gitlab_usage_ping_worker.rb
+++ b/app/workers/gitlab_usage_ping_worker.rb
@@ -6,10 +6,16 @@ class GitlabUsagePingWorker
   include ApplicationWorker
   include CronjobQueue
 
+  # Retry for up to approximately three hours then give up.
+  sidekiq_options retry: 10, dead: false
+
   def perform
     # Multiple Sidekiq workers could run this. We should only do this at most once a day.
     return unless try_obtain_lease
 
+    # Splay the request over a minute to avoid thundering herd problems.
+    sleep(rand(0.0..60.0).round(3))
+
     SubmitUsagePingService.new.execute
   end
 
diff --git a/changelogs/unreleased/bjk-usage_ping.yml b/changelogs/unreleased/bjk-usage_ping.yml
new file mode 100644
index 00000000000..dee6c1ad291
--- /dev/null
+++ b/changelogs/unreleased/bjk-usage_ping.yml
@@ -0,0 +1,5 @@
+---
+title: Update usage ping cron behavior
+merge_request: 30842
+author:
+type: performance
diff --git a/config/settings.rb b/config/settings.rb
index da459afcce2..8756c120645 100644
--- a/config/settings.rb
+++ b/config/settings.rb
@@ -1,4 +1,5 @@
 require 'settingslogic'
+require 'digest/md5'
 
 # We can not use `Rails.root` here, as this file might be loaded without the
 # full Rails environment being loaded. We can not use `require_relative` either,
@@ -170,14 +171,17 @@ class Settings < Settingslogic
       URI.parse(url_without_path).host
     end
 
-    # Runs every minute in a random ten-minute period on Sundays, to balance the
-    # load on the server receiving these pings. The usage ping is safe to run
-    # multiple times because of a 24 hour exclusive lock.
+    # Runs at a random time of day on a consistent day of the week based on
+    # the instance UUID. This is to balance the load on the service receiving
+    # these pings. The sidekiq job handles temporary http failures.
     def cron_for_usage_ping
       hour = rand(24)
-      minute = rand(6)
+      minute = rand(60)
+      # Set a default UUID for the case when the UUID hasn't been initialized.
+      uuid = Gitlab::CurrentSettings.uuid || 'uuid-not-set'
+      day_of_week = Digest::MD5.hexdigest(uuid).to_i(16) % 7
 
-      "#{minute}0-#{minute}9 #{hour} * * 0"
+      "#{minute} #{hour} * * #{day_of_week}"
     end
   end
 end
-- 
cgit v1.2.1