summaryrefslogtreecommitdiff
path: root/config
diff options
context:
space:
mode:
authorAndrew Newdigate <andrew@gitlab.com>2018-10-20 19:00:19 +0100
committerAndrew Newdigate <andrew@gitlab.com>2018-10-25 17:50:15 +0100
commit1065f8ce7a261dff5a3077be46405343141733df (patch)
tree92669873cb55a448de6a581a86d970148762d210 /config
parent605e952e39ddad4efa786ebc06a3175727563db5 (diff)
downloadgitlab-ce-1065f8ce7a261dff5a3077be46405343141733df.tar.gz
Add experimental support for Pumaan-multithreading
This allows us (and others) to test drive Puma without it affecting all users. Puma can be enabled by setting the environment variable "EXPERIMENTAL_PUMA" to a non empty value.
Diffstat (limited to 'config')
-rw-r--r--config/initializers/7_prometheus_metrics.rb22
-rw-r--r--config/initializers/8_metrics.rb4
-rw-r--r--config/initializers/active_record_lifecycle.rb23
-rw-r--r--config/initializers/macos.rb13
-rw-r--r--config/initializers/rbtrace.rb9
-rw-r--r--config/initializers/sidekiq.rb2
-rw-r--r--config/puma.example.development.rb77
-rw-r--r--config/unicorn.rb.example37
-rw-r--r--config/unicorn.rb.example.development67
9 files changed, 200 insertions, 54 deletions
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index 146c4b1e024..8052880cc3d 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -26,9 +26,25 @@ Sidekiq.configure_server do |config|
end
if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
- unless Sidekiq.server?
- Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
+ Gitlab::Cluster::LifecycleEvents.on_worker_start do
+ defined?(::Prometheus::Client.reinitialize_on_pid_change) && Prometheus::Client.reinitialize_on_pid_change
+
+ unless Sidekiq.server?
+ Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
+ end
+
+ Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start
end
+end
- Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start
+Gitlab::Cluster::LifecycleEvents.on_master_restart do
+ # The following is necessary to ensure stale Prometheus metrics don't
+ # accumulate over time. It needs to be done in this hook as opposed to
+ # inside an init script to ensure metrics files aren't deleted after new
+ # unicorn workers start after a SIGUSR2 is received.
+ prometheus_multiproc_dir = ENV['prometheus_multiproc_dir']
+ if prometheus_multiproc_dir
+ old_metrics = Dir[File.join(prometheus_multiproc_dir, '*.db')]
+ FileUtils.rm_rf(old_metrics)
+ end
end
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index eccf82ab8dc..c8d261d415e 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -158,7 +158,9 @@ if Gitlab::Metrics.enabled? && !Rails.env.test?
GC::Profiler.enable
- Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start
+ Gitlab::Cluster::LifecycleEvents.on_worker_start do
+ Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start
+ end
module TrackNewRedisConnections
def connect(*args)
diff --git a/config/initializers/active_record_lifecycle.rb b/config/initializers/active_record_lifecycle.rb
new file mode 100644
index 00000000000..7fa37121efc
--- /dev/null
+++ b/config/initializers/active_record_lifecycle.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Don't handle sidekiq configuration as it
+# has its own special active record configuration here
+if defined?(ActiveRecord::Base) && !Sidekiq.server?
+ Gitlab::Cluster::LifecycleEvents.on_worker_start do
+ ActiveSupport.on_load(:active_record) do
+ ActiveRecord::Base.establish_connection
+
+ Rails.logger.debug("ActiveRecord connection established")
+ end
+ end
+end
+
+if defined?(ActiveRecord::Base)
+ Gitlab::Cluster::LifecycleEvents.on_before_fork do
+ # the following is highly recommended for Rails + "preload_app true"
+ # as there's no need for the master process to hold a connection
+ ActiveRecord::Base.connection.disconnect!
+
+ Rails.logger.debug("ActiveRecord connection disconnected")
+ end
+end
diff --git a/config/initializers/macos.rb b/config/initializers/macos.rb
new file mode 100644
index 00000000000..f410af6ed47
--- /dev/null
+++ b/config/initializers/macos.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+if /darwin/ =~ RUBY_PLATFORM
+ Gitlab::Cluster::LifecycleEvents.on_before_fork do
+ require 'fiddle'
+
+ # Dynamically load Foundation.framework, ~implicitly~ initialising
+ # the Objective-C runtime before any forking happens in Unicorn
+ #
+ # From https://bugs.ruby-lang.org/issues/14009
+ Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation'
+ end
+end
diff --git a/config/initializers/rbtrace.rb b/config/initializers/rbtrace.rb
new file mode 100644
index 00000000000..6a1b71bf4bd
--- /dev/null
+++ b/config/initializers/rbtrace.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+if ENV['ENABLE_RBTRACE']
+ Gitlab::Cluster::LifecycleEvents.on_worker_start do
+ # Unicorn clears out signals before it forks, so rbtrace won't work
+ # unless it is enabled after the fork.
+ require 'rbtrace'
+ end
+end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index bc6b7aed6aa..565efc858d1 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -14,8 +14,6 @@ Sidekiq.default_worker_options = { retry: 3 }
enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
Sidekiq.configure_server do |config|
- require 'rbtrace' if ENV['ENABLE_RBTRACE']
-
config.redis = queues_config_hash
config.server_middleware do |chain|
diff --git a/config/puma.example.development.rb b/config/puma.example.development.rb
new file mode 100644
index 00000000000..490c940077a
--- /dev/null
+++ b/config/puma.example.development.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# -----------------------------------------------------------------------
+# This file is used by the GDK to generate a default config/puma.rb file
+# Note that `/home/git` will be substituted for the actual GDK root
+# directory when this file is generated
+# -----------------------------------------------------------------------
+
+# Load "path" as a rackup file.
+#
+# The default is "config.ru".
+#
+rackup 'config.ru'
+pidfile '/home/git/gitlab/tmp/pids/puma.pid'
+state_path '/home/git/gitlab/tmp/pids/puma.state'
+
+stdout_redirect '/home/git/gitlab/log/puma.stdout.log',
+ '/home/git/gitlab/log/puma.stderr.log',
+ true
+
+# Configure "min" to be the minimum number of threads to use to answer
+# requests and "max" the maximum.
+#
+# The default is "0, 16".
+#
+threads 1, 4
+
+# By default, workers accept all requests and queue them to pass to handlers.
+# When false, workers accept the number of simultaneous requests configured.
+#
+# Queueing requests generally improves performance, but can cause deadlocks if
+# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
+#
+# When set to false this may require a reverse proxy to handle slow clients and
+# queue requests before they reach puma. This is due to disabling HTTP keepalive
+queue_requests false
+
+# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
+# accepted protocols.
+bind 'unix:///home/git/gitlab.socket'
+
+workers 2
+
+require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
+require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer"
+
+on_restart do
+ # Signal application hooks that we're about to restart
+ Gitlab::Cluster::LifecycleEvents.do_master_restart
+end
+
+before_fork do
+ # Signal to the puma killer
+ Gitlab::Cluster::PumaWorkerKillerInitializer.start @config.options unless ENV['DISABLE_PUMA_WORKER_KILLER']
+
+ # Signal application hooks that we're about to fork
+ Gitlab::Cluster::LifecycleEvents.do_before_fork
+end
+
+Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options
+on_worker_boot do
+ # Signal application hooks of worker start
+ Gitlab::Cluster::LifecycleEvents.do_worker_start
+end
+
+# Preload the application before starting the workers; this conflicts with
+# phased restart feature. (off by default)
+
+preload_app!
+
+tag 'gitlab-puma-worker'
+
+# Verifies that all workers have checked in to the master process within
+# the given timeout. If not the worker process will be restarted. Default
+# value is 60 seconds.
+#
+worker_timeout 60
diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example
index e06cce3e97a..4637eb8bc6e 100644
--- a/config/unicorn.rb.example
+++ b/config/unicorn.rb.example
@@ -81,22 +81,16 @@ preload_app true
# fast LAN.
check_client_connection false
+require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
+
before_exec do |server|
- # The following is necessary to ensure stale Prometheus metrics don't
- # accumulate over time. It needs to be done in this hook as opposed to
- # inside an init script to ensure metrics files aren't deleted after new
- # unicorn workers start after a SIGUSR2 is received.
- if ENV['prometheus_multiproc_dir']
- old_metrics = Dir[File.join(ENV['prometheus_multiproc_dir'], '*.db')]
- FileUtils.rm_rf(old_metrics)
- end
+ # Signal application hooks that we're about to restart
+ Gitlab::Cluster::LifecycleEvents.do_master_restart
end
before_fork do |server, worker|
- # the following is highly recommended for Rails + "preload_app true"
- # as there's no need for the master process to hold a connection
- defined?(ActiveRecord::Base) &&
- ActiveRecord::Base.connection.disconnect!
+ # Signal application hooks that we're about to fork
+ Gitlab::Cluster::LifecycleEvents.do_before_fork
# The following is only recommended for memory/DB-constrained
# installations. It is not needed if your system can house
@@ -124,25 +118,10 @@ before_fork do |server, worker|
end
after_fork do |server, worker|
- # Unicorn clears out signals before it forks, so rbtrace won't work
- # unless it is enabled after the fork.
- require 'rbtrace' if ENV['ENABLE_RBTRACE']
+ # Signal application hooks of worker start
+ Gitlab::Cluster::LifecycleEvents.do_worker_start
# per-process listener ports for debugging/admin/migrations
# addr = "127.0.0.1:#{9293 + worker.nr}"
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
-
- # the following is *required* for Rails + "preload_app true",
- defined?(ActiveRecord::Base) &&
- ActiveRecord::Base.establish_connection
-
- # reset prometheus client, this will cause any opened metrics files to be closed
- defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
- Prometheus::Client.reinitialize_on_pid_change
-
- # if preload_app is true, then you may also want to check and
- # restart any other shared sockets/descriptors such as Memcached,
- # and Redis. TokyoCabinet file handles are safe to reuse
- # between any number of forked children (assuming your kernel
- # correctly implements pread()/pwrite() system calls)
end
diff --git a/config/unicorn.rb.example.development b/config/unicorn.rb.example.development
index f31df66015a..f7541bb9d55 100644
--- a/config/unicorn.rb.example.development
+++ b/config/unicorn.rb.example.development
@@ -1,32 +1,61 @@
+# frozen_string_literal: true
+
+# -------------------------------------------------------------------------
+# This file is used by the GDK to generate a default config/unicorn.rb file
+# Note that `/home/git` will be substituted for the actual GDK root
+# directory when this file is generated
+# -------------------------------------------------------------------------
+
worker_processes 2
timeout 60
+listen '/home/git/gitlab.socket'
+
preload_app true
check_client_connection false
+require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
+
+before_exec do |server|
+ # Signal application hooks that we're about to restart
+ Gitlab::Cluster::LifecycleEvents.do_master_restart
+end
+
before_fork do |server, worker|
- # the following is highly recommended for Rails + "preload_app true"
- # as there's no need for the master process to hold a connection
- defined?(ActiveRecord::Base) &&
- ActiveRecord::Base.connection.disconnect!
-
- if /darwin/ =~ RUBY_PLATFORM
- require 'fiddle'
-
- # Dynamically load Foundation.framework, ~implicitly~ initialising
- # the Objective-C runtime before any forking happens in Unicorn
- #
- # From https://bugs.ruby-lang.org/issues/14009
- Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation'
+ # Signal application hooks that we're about to fork
+ Gitlab::Cluster::LifecycleEvents.do_before_fork
+
+ # The following is only recommended for memory/DB-constrained
+ # installations. It is not needed if your system can house
+ # twice as many worker_processes as you have configured.
+ #
+ # This allows a new master process to incrementally
+ # phase out the old master process with SIGTTOU to avoid a
+ # thundering herd (especially in the "preload_app false" case)
+ # when doing a transparent upgrade. The last worker spawned
+ # will then kill off the old master process with a SIGQUIT.
+ old_pid = "#{server.config[:pid]}.oldbin"
+ if old_pid != server.pid
+ begin
+ sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
+ Process.kill(sig, File.read(old_pid).to_i)
+ rescue Errno::ENOENT, Errno::ESRCH
+ end
end
+ #
+ # Throttle the master from forking too quickly by sleeping. Due
+ # to the implementation of standard Unix signal handlers, this
+ # helps (but does not completely) prevent identical, repeated signals
+ # from being lost when the receiving process is busy.
+ # sleep 1
end
after_fork do |server, worker|
- # Unicorn clears out signals before it forks, so rbtrace won't work
- # unless it is enabled after the fork.
- require 'rbtrace' if ENV['ENABLE_RBTRACE']
+ # Signal application hooks of worker start
+ Gitlab::Cluster::LifecycleEvents.do_worker_start
- # the following is *required* for Rails + "preload_app true",
- defined?(ActiveRecord::Base) &&
- ActiveRecord::Base.establish_connection
+ # per-process listener ports for debugging/admin/migrations
+ # addr = "127.0.0.1:#{9293 + worker.nr}"
+ # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
end
+