summaryrefslogtreecommitdiff
path: root/sidekiq_cluster
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2022-03-18 20:02:30 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2022-03-18 20:02:30 +0000
commit41fe97390ceddf945f3d967b8fdb3de4c66b7dea (patch)
tree9c8d89a8624828992f06d892cd2f43818ff5dcc8 /sidekiq_cluster
parent0804d2dc31052fb45a1efecedc8e06ce9bc32862 (diff)
downloadgitlab-ce-41fe97390ceddf945f3d967b8fdb3de4c66b7dea.tar.gz
Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42
Diffstat (limited to 'sidekiq_cluster')
-rw-r--r--sidekiq_cluster/cli.rb109
-rw-r--r--sidekiq_cluster/sidekiq_cluster.rb10
2 files changed, 41 insertions, 78 deletions
diff --git a/sidekiq_cluster/cli.rb b/sidekiq_cluster/cli.rb
index 2feb77601b8..e04f5ab1d42 100644
--- a/sidekiq_cluster/cli.rb
+++ b/sidekiq_cluster/cli.rb
@@ -13,7 +13,7 @@ require_relative '../lib/gitlab/utils'
require_relative '../lib/gitlab/sidekiq_config/cli_methods'
require_relative '../lib/gitlab/sidekiq_config/worker_matcher'
require_relative '../lib/gitlab/sidekiq_logging/json_formatter'
-require_relative '../lib/gitlab/process_management'
+require_relative '../metrics_server/dependencies'
require_relative '../metrics_server/metrics_server'
require_relative 'sidekiq_cluster'
@@ -38,8 +38,7 @@ module Gitlab
@metrics_dir = ENV["prometheus_multiproc_dir"] || File.absolute_path("tmp/prometheus_multiproc_dir/sidekiq")
@pid = nil
@interval = 5
- @alive = true
- @processes = []
+ @soft_timeout_seconds = DEFAULT_SOFT_TIMEOUT_SECONDS
@logger = Logger.new(log_output)
@logger.formatter = ::Gitlab::SidekiqLogging::JSONFormatter.new
@rails_path = Dir.pwd
@@ -103,95 +102,64 @@ module Gitlab
@logger.info("Starting cluster with #{queue_groups.length} processes")
end
- start_metrics_server(wipe_metrics_dir: true)
+ start_and_supervise_workers(queue_groups)
+ end
- @processes = SidekiqCluster.start(
+ def start_and_supervise_workers(queue_groups)
+ worker_pids = SidekiqCluster.start(
queue_groups,
env: @environment,
directory: @rails_path,
max_concurrency: @max_concurrency,
min_concurrency: @min_concurrency,
dryrun: @dryrun,
- timeout: soft_timeout_seconds
+ timeout: @soft_timeout_seconds
)
return if @dryrun
- write_pid
- trap_signals
- start_loop
- end
-
- def write_pid
ProcessManagement.write_pid(@pid) if @pid
- end
-
- def soft_timeout_seconds
- @soft_timeout_seconds || DEFAULT_SOFT_TIMEOUT_SECONDS
- end
-
- # The amount of time it'll wait for killing the alive Sidekiq processes.
- def hard_timeout_seconds
- soft_timeout_seconds + DEFAULT_HARD_TIMEOUT_SECONDS
- end
-
- def monotonic_time
- Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
- end
-
- def continue_waiting?(deadline)
- ProcessManagement.any_alive?(@processes) && monotonic_time < deadline
- end
-
- def hard_stop_stuck_pids
- ProcessManagement.signal_processes(ProcessManagement.pids_alive(@processes), "-KILL")
- end
-
- def wait_for_termination
- deadline = monotonic_time + hard_timeout_seconds
- sleep(CHECK_TERMINATE_INTERVAL_SECONDS) while continue_waiting?(deadline)
- hard_stop_stuck_pids
- end
-
- def trap_signals
- ProcessManagement.trap_signals(TERMINATE_SIGNALS) do |signal|
- @alive = false
- ProcessManagement.signal_processes(@processes, signal)
- wait_for_termination
- end
-
- ProcessManagement.trap_signals(FORWARD_SIGNALS) do |signal|
- ProcessManagement.signal_processes(@processes, signal)
- end
- end
+ supervisor = SidekiqProcessSupervisor.instance(
+ health_check_interval_seconds: @interval,
+ terminate_timeout_seconds: @soft_timeout_seconds + TIMEOUT_GRACE_PERIOD_SECONDS,
+ term_signals: TERMINATE_SIGNALS,
+ forwarded_signals: FORWARD_SIGNALS,
+ synchronous: true
+ )
- def start_loop
- while @alive
- sleep(@interval)
+ metrics_server_pid = start_metrics_server
- if metrics_server_enabled? && ProcessManagement.process_died?(@metrics_server_pid)
- @logger.warn('Metrics server went away')
- start_metrics_server(wipe_metrics_dir: false)
- end
+ all_pids = worker_pids + Array(metrics_server_pid)
- unless ProcessManagement.all_alive?(@processes)
- # If a child process died we'll just terminate the whole cluster. It's up to
- # runit and such to then restart the cluster.
+ supervisor.supervise(all_pids) do |dead_pids|
+ # If we're not in the process of shutting down the cluster,
+ # and the metrics server died, restart it.
+ if supervisor.alive && dead_pids.include?(metrics_server_pid)
+ @logger.info('Sidekiq metrics server terminated, restarting...')
+ metrics_server_pid = restart_metrics_server(wipe_metrics_dir: false)
+ all_pids = worker_pids + Array(metrics_server_pid)
+ else
+ # If a worker process died we'll just terminate the whole cluster.
+ # We let an external system (runit, kubernetes) handle the restart.
@logger.info('A worker terminated, shutting down the cluster')
- stop_metrics_server
- ProcessManagement.signal_processes(@processes, :TERM)
- break
+ ProcessManagement.signal_processes(all_pids - dead_pids, :TERM)
+ # Signal supervisor not to respawn workers and shut down.
+ []
end
end
end
- def start_metrics_server(wipe_metrics_dir: false)
+ def start_metrics_server
return unless metrics_server_enabled?
+ restart_metrics_server(wipe_metrics_dir: true)
+ end
+
+ def restart_metrics_server(wipe_metrics_dir: false)
@logger.info("Starting metrics server on port #{sidekiq_exporter_port}")
- @metrics_server_pid = MetricsServer.fork(
+ MetricsServer.fork(
'sidekiq',
metrics_dir: @metrics_dir,
wipe_metrics_dir: wipe_metrics_dir,
@@ -225,13 +193,6 @@ module Gitlab
!@dryrun && sidekiq_exporter_enabled? && exporter_has_a_unique_port?
end
- def stop_metrics_server
- return unless @metrics_server_pid
-
- @logger.info("Stopping metrics server (PID #{@metrics_server_pid})")
- ProcessManagement.signal(@metrics_server_pid, :TERM)
- end
-
def option_parser
OptionParser.new do |opt|
opt.banner = "#{File.basename(__FILE__)} [QUEUE,QUEUE] [QUEUE] ... [OPTIONS]"
diff --git a/sidekiq_cluster/sidekiq_cluster.rb b/sidekiq_cluster/sidekiq_cluster.rb
index c5139ab8874..c68cbe7c163 100644
--- a/sidekiq_cluster/sidekiq_cluster.rb
+++ b/sidekiq_cluster/sidekiq_cluster.rb
@@ -4,8 +4,6 @@ require_relative '../lib/gitlab/process_management'
module Gitlab
module SidekiqCluster
- CHECK_TERMINATE_INTERVAL_SECONDS = 1
-
# How long to wait when asking for a clean termination.
# It maps the Sidekiq default timeout:
# https://github.com/mperham/sidekiq/wiki/Signals#term
@@ -14,8 +12,12 @@ module Gitlab
# is given through arguments.
DEFAULT_SOFT_TIMEOUT_SECONDS = 25
- # After surpassing the soft timeout.
- DEFAULT_HARD_TIMEOUT_SECONDS = 5
+ # Additional time granted after surpassing the soft timeout
+ # before we kill the process.
+ TIMEOUT_GRACE_PERIOD_SECONDS = 5
+
+ # The singleton instance used to supervise cluster processes.
+ SidekiqProcessSupervisor = Class.new(Gitlab::ProcessSupervisor)
# Starts Sidekiq workers for the pairs of processes.
#