From f4e94c6ff7b2b5e96edb56d30737687becc4a1a8 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 13 Oct 2017 12:22:38 +0100 Subject: Make usage ping scheduling more robust On GitLab.com, we see that cron jobs sometimes don't run. This appears to be because the process that polls for current cron jobs only has a 60 second validity period, so if (for whatever reason) it misses those 60 seconds, we have to wait until the next execution. For the usage ping, this is particularly problematic, as that's only scheduled to run once a week. Changing it to run for every minute in a 10 minute period should work around this, if the above diagnosis is correct. The job itself obtains an exclusive lease for 24 hours, so rescheduling in quick succession is safe. --- ...17-gitlabusagepingworker-is-not-running-on-gitlab-com.yml | 5 +++++ config/initializers/1_settings.rb | 12 +++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 changelogs/unreleased/39017-gitlabusagepingworker-is-not-running-on-gitlab-com.yml diff --git a/changelogs/unreleased/39017-gitlabusagepingworker-is-not-running-on-gitlab-com.yml b/changelogs/unreleased/39017-gitlabusagepingworker-is-not-running-on-gitlab-com.yml new file mode 100644 index 00000000000..89506f88637 --- /dev/null +++ b/changelogs/unreleased/39017-gitlabusagepingworker-is-not-running-on-gitlab-com.yml @@ -0,0 +1,5 @@ +--- +title: Make usage ping scheduling more robust +merge_request: +author: +type: fixed diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index a4b7c1a3919..b790df565c6 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -113,12 +113,14 @@ class Settings < Settingslogic URI.parse(url_without_path).host end - # Random cron time every Sunday to load balance usage pings - def cron_random_weekly_time + # Runs every minute in a random ten-minute period on Sundays, to balance the + # load on the server receiving these pings. The usage ping is safe to run + # multiple times because of a 24 hour exclusive lock. + def cron_for_usage_ping hour = rand(24) - minute = rand(60) + minute = rand(6) - "#{minute} #{hour} * * 0" + "#{minute}0-#{minute}9 #{hour} * * 0" end end end @@ -398,7 +400,7 @@ Settings.cron_jobs['stuck_import_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_import_jobs_worker']['cron'] ||= '15 * * * *' Settings.cron_jobs['stuck_import_jobs_worker']['job_class'] = 'StuckImportJobsWorker' Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({}) -Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_random_weekly_time) +Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_for_usage_ping) Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker' Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({}) -- cgit v1.2.1