Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42

2 files changed, 41 insertions, 78 deletions

diff --git a/sidekiq_cluster/cli.rb b/sidekiq_cluster/cli.rb
index 2feb77601b8..e04f5ab1d42 100644
--- a/sidekiq_cluster/cli.rb
+++ b/sidekiq_cluster/cli.rb
@@ -13,7 +13,7 @@ require_relative '../lib/gitlab/utils'
 require_relative '../lib/gitlab/sidekiq_config/cli_methods'
 require_relative '../lib/gitlab/sidekiq_config/worker_matcher'
 require_relative '../lib/gitlab/sidekiq_logging/json_formatter'
-require_relative '../lib/gitlab/process_management'
+require_relative '../metrics_server/dependencies'
 require_relative '../metrics_server/metrics_server'
 require_relative 'sidekiq_cluster'
 
@@ -38,8 +38,7 @@ module Gitlab
         @metrics_dir = ENV["prometheus_multiproc_dir"] || File.absolute_path("tmp/prometheus_multiproc_dir/sidekiq")
         @pid = nil
         @interval = 5
-        @alive = true
-        @processes = []
+        @soft_timeout_seconds = DEFAULT_SOFT_TIMEOUT_SECONDS
         @logger = Logger.new(log_output)
         @logger.formatter = ::Gitlab::SidekiqLogging::JSONFormatter.new
         @rails_path = Dir.pwd
@@ -103,95 +102,64 @@ module Gitlab
           @logger.info("Starting cluster with #{queue_groups.length} processes")
         end
 
-        start_metrics_server(wipe_metrics_dir: true)
+        start_and_supervise_workers(queue_groups)
+      end
 
-        @processes = SidekiqCluster.start(
+      def start_and_supervise_workers(queue_groups)
+        worker_pids = SidekiqCluster.start(
           queue_groups,
           env: @environment,
           directory: @rails_path,
           max_concurrency: @max_concurrency,
           min_concurrency: @min_concurrency,
           dryrun: @dryrun,
-          timeout: soft_timeout_seconds
+          timeout: @soft_timeout_seconds
         )
 
         return if @dryrun
 
-        write_pid
-        trap_signals
-        start_loop
-      end
-
-      def write_pid
         ProcessManagement.write_pid(@pid) if @pid
-      end
-
-      def soft_timeout_seconds
-        @soft_timeout_seconds || DEFAULT_SOFT_TIMEOUT_SECONDS
-      end
-
-      # The amount of time it'll wait for killing the alive Sidekiq processes.
-      def hard_timeout_seconds
-        soft_timeout_seconds + DEFAULT_HARD_TIMEOUT_SECONDS
-      end
-
-      def monotonic_time
-        Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
-      end
-
-      def continue_waiting?(deadline)
-        ProcessManagement.any_alive?(@processes) && monotonic_time < deadline
-      end
-
-      def hard_stop_stuck_pids
-        ProcessManagement.signal_processes(ProcessManagement.pids_alive(@processes), "-KILL")
-      end
-
-      def wait_for_termination
-        deadline = monotonic_time + hard_timeout_seconds
-        sleep(CHECK_TERMINATE_INTERVAL_SECONDS) while continue_waiting?(deadline)
 
-        hard_stop_stuck_pids
-      end
-
-      def trap_signals
-        ProcessManagement.trap_signals(TERMINATE_SIGNALS) do |signal|
-          @alive = false
-          ProcessManagement.signal_processes(@processes, signal)
-          wait_for_termination
-        end
-
-        ProcessManagement.trap_signals(FORWARD_SIGNALS) do |signal|
-          ProcessManagement.signal_processes(@processes, signal)
-        end
-      end
+        supervisor = SidekiqProcessSupervisor.instance(
+          health_check_interval_seconds: @interval,
+          terminate_timeout_seconds: @soft_timeout_seconds + TIMEOUT_GRACE_PERIOD_SECONDS,
+          term_signals: TERMINATE_SIGNALS,
+          forwarded_signals: FORWARD_SIGNALS,
+          synchronous: true
+        )
 
-      def start_loop
-        while @alive
-          sleep(@interval)
+        metrics_server_pid = start_metrics_server
 
-          if metrics_server_enabled? && ProcessManagement.process_died?(@metrics_server_pid)
-            @logger.warn('Metrics server went away')
-            start_metrics_server(wipe_metrics_dir: false)
-          end
+        all_pids = worker_pids + Array(metrics_server_pid)
 
-          unless ProcessManagement.all_alive?(@processes)
-            # If a child process died we'll just terminate the whole cluster. It's up to
-            # runit and such to then restart the cluster.
+        supervisor.supervise(all_pids) do |dead_pids|
+          # If we're not in the process of shutting down the cluster,
+          # and the metrics server died, restart it.
+          if supervisor.alive && dead_pids.include?(metrics_server_pid)
+            @logger.info('Sidekiq metrics server terminated, restarting...')
+            metrics_server_pid = restart_metrics_server(wipe_metrics_dir: false)
+            all_pids = worker_pids + Array(metrics_server_pid)
+          else
+            # If a worker process died we'll just terminate the whole cluster.
+            # We let an external system (runit, kubernetes) handle the restart.
             @logger.info('A worker terminated, shutting down the cluster')
 
-            stop_metrics_server
-            ProcessManagement.signal_processes(@processes, :TERM)
-            break
+            ProcessManagement.signal_processes(all_pids - dead_pids, :TERM)
+            # Signal supervisor not to respawn workers and shut down.
+            []
           end
         end
       end
 
-      def start_metrics_server(wipe_metrics_dir: false)
+      def start_metrics_server
         return unless metrics_server_enabled?
 
+        restart_metrics_server(wipe_metrics_dir: true)
+      end
+
+      def restart_metrics_server(wipe_metrics_dir: false)
         @logger.info("Starting metrics server on port #{sidekiq_exporter_port}")
-        @metrics_server_pid = MetricsServer.fork(
+        MetricsServer.fork(
           'sidekiq',
           metrics_dir: @metrics_dir,
           wipe_metrics_dir: wipe_metrics_dir,
@@ -225,13 +193,6 @@ module Gitlab
         !@dryrun && sidekiq_exporter_enabled? && exporter_has_a_unique_port?
       end
 
-      def stop_metrics_server
-        return unless @metrics_server_pid
-
-        @logger.info("Stopping metrics server (PID #{@metrics_server_pid})")
-        ProcessManagement.signal(@metrics_server_pid, :TERM)
-      end
-
       def option_parser
         OptionParser.new do |opt|
           opt.banner = "#{File.basename(__FILE__)} [QUEUE,QUEUE] [QUEUE] ... [OPTIONS]"
diff --git a/sidekiq_cluster/sidekiq_cluster.rb b/sidekiq_cluster/sidekiq_cluster.rb
index c5139ab8874..c68cbe7c163 100644
--- a/sidekiq_cluster/sidekiq_cluster.rb
+++ b/sidekiq_cluster/sidekiq_cluster.rb
@@ -4,8 +4,6 @@ require_relative '../lib/gitlab/process_management'
 
 module Gitlab
   module SidekiqCluster
-    CHECK_TERMINATE_INTERVAL_SECONDS = 1
-
     # How long to wait when asking for a clean termination.
     # It maps the Sidekiq default timeout:
     # https://github.com/mperham/sidekiq/wiki/Signals#term
@@ -14,8 +12,12 @@ module Gitlab
     # is given through arguments.
     DEFAULT_SOFT_TIMEOUT_SECONDS = 25
 
-    # After surpassing the soft timeout.
-    DEFAULT_HARD_TIMEOUT_SECONDS = 5
+    # Additional time granted after surpassing the soft timeout
+    # before we kill the process.
+    TIMEOUT_GRACE_PERIOD_SECONDS = 5
+
+    # The singleton instance used to supervise cluster processes.
+    SidekiqProcessSupervisor = Class.new(Gitlab::ProcessSupervisor)
 
     # Starts Sidekiq workers for the pairs of processes.
     #