diff options
author | Robert Speicher <robert@gitlab.com> | 2016-04-08 22:26:53 +0000 |
---|---|---|
committer | Robert Speicher <robert@gitlab.com> | 2016-04-08 22:26:53 +0000 |
commit | 93571c433584d7d183c8d0555a4d210d0ae75e16 (patch) | |
tree | 91662554ff7c2245cf8e0b99b3963fd21fb0dfb6 | |
parent | 833808d737058ff25b34f6bd5eb4d4989c43e49d (diff) | |
parent | cf669551f69edd66913d22c96cf1de1302e7990e (diff) | |
download | gitlab-ce-93571c433584d7d183c8d0555a4d210d0ae75e16.tar.gz |
Merge branch 'redis-connection-pool' into 'master'
Redis connection pool
Split from https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3232
Having an easily accessible Redis connection pool allows us to do more
cool stuff with Redis in GitLab (instead of having to go through e.g.
the Rails cache).
See merge request !3521
-rw-r--r-- | Gemfile | 5 | ||||
-rw-r--r-- | Gemfile.lock | 1 | ||||
-rw-r--r-- | config/application.rb | 8 | ||||
-rw-r--r-- | config/initializers/session_store.rb | 2 | ||||
-rw-r--r-- | config/initializers/sidekiq.rb | 4 | ||||
-rw-r--r-- | config/mail_room.yml | 4 | ||||
-rw-r--r-- | lib/gitlab/exclusive_lease.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/redis.rb | 48 | ||||
-rw-r--r-- | lib/gitlab/redis_config.rb | 30 | ||||
-rw-r--r-- | lib/tasks/cache.rake | 25 |
10 files changed, 77 insertions, 54 deletions
@@ -149,6 +149,10 @@ gem 'version_sorter', '~> 2.0.0' # Cache gem "redis-rails", '~> 4.0.0' +# Redis +gem 'redis', '~> 3.2' +gem 'connection_pool', '~> 2.0' + # Campfire integration gem 'tinder', '~> 1.10.0' @@ -229,7 +233,6 @@ group :metrics do gem 'allocations', '~> 1.0', require: false, platform: :mri gem 'method_source', '~> 0.8', require: false gem 'influxdb', '~> 0.2', require: false - gem 'connection_pool', '~> 2.0', require: false end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 1ba8d748db1..a30706a4318 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -999,6 +999,7 @@ DEPENDENCIES rdoc (~> 3.6) recaptcha redcarpet (~> 3.3.3) + redis (~> 3.2) redis-namespace redis-rails (~> 4.0.0) request_store (~> 1.3.0) diff --git a/config/application.rb b/config/application.rb index 5a0ac70aa2a..2e2ed48db07 100644 --- a/config/application.rb +++ b/config/application.rb @@ -4,11 +4,9 @@ require 'rails/all' require 'devise' I18n.config.enforce_available_locales = false Bundler.require(:default, Rails.env) -require_relative '../lib/gitlab/redis_config' +require_relative '../lib/gitlab/redis' module Gitlab - REDIS_CACHE_NAMESPACE = 'cache:gitlab' - class Application < Rails::Application # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers @@ -69,8 +67,8 @@ module Gitlab end end - redis_config_hash = Gitlab::RedisConfig.redis_store_options - redis_config_hash[:namespace] = REDIS_CACHE_NAMESPACE + redis_config_hash = Gitlab::Redis.redis_store_options + redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever config.cache_store = :redis_store, redis_config_hash diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 3da5d46be92..70285255877 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -13,7 +13,7 @@ end if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - redis_config = Gitlab::RedisConfig.redis_store_options + redis_config = Gitlab::Redis.redis_store_options redis_config[:namespace] = 'session:gitlab' Gitlab::Application.config.session_store( diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index cc83137745a..9182d929809 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -2,7 +2,7 @@ SIDEKIQ_REDIS_NAMESPACE = 'resque:gitlab' Sidekiq.configure_server do |config| config.redis = { - url: Gitlab::RedisConfig.url, + url: Gitlab::Redis.url, namespace: SIDEKIQ_REDIS_NAMESPACE } @@ -29,7 +29,7 @@ end Sidekiq.configure_client do |config| config.redis = { - url: Gitlab::RedisConfig.url, + url: Gitlab::Redis.url, namespace: SIDEKIQ_REDIS_NAMESPACE } end diff --git a/config/mail_room.yml b/config/mail_room.yml index 60257329f3e..761a32adb9e 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -2,7 +2,7 @@ <% require "yaml" require "json" -require_relative "lib/gitlab/redis_config" +require_relative "lib/gitlab/redis" rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development" @@ -18,7 +18,7 @@ if File.exists?(config_file) config['mailbox'] = "inbox" if config['mailbox'].nil? if config['enabled'] && config['address'] - redis_url = Gitlab::RedisConfig.new(rails_env).url + redis_url = Gitlab::Redis.new(rails_env).url %> - :host: <%= config['host'].to_json %> diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index c73eca832d7..c2260a5f7ac 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -43,7 +43,9 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + Gitlab::Redis.with do |redis| + !!redis.set(redis_key, '1', nx: true, ex: @timeout) + end end # No #cancel method. See comments above! diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb new file mode 100644 index 00000000000..319447669dc --- /dev/null +++ b/lib/gitlab/redis.rb @@ -0,0 +1,48 @@ +module Gitlab + class Redis + CACHE_NAMESPACE = 'cache:gitlab' + + attr_reader :url + + # To be thread-safe we must be careful when writing the class instance + # variables @url and @pool. Because @pool depends on @url we need two + # mutexes to prevent deadlock. + URL_MUTEX = Mutex.new + POOL_MUTEX = Mutex.new + private_constant :URL_MUTEX, :POOL_MUTEX + + def self.url + @url || URL_MUTEX.synchronize { @url = new.url } + end + + def self.with + if @pool.nil? + POOL_MUTEX.synchronize do + @pool = ConnectionPool.new { ::Redis.new(url: url) } + end + end + @pool.with { |redis| yield redis } + end + + def self.redis_store_options + url = new.url + redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) + # Redis::Store does not handle Unix sockets well, so let's do it for them + redis_uri = URI.parse(url) + if redis_uri.scheme == 'unix' + redis_config_hash[:path] = redis_uri.path + end + redis_config_hash + end + + def initialize(rails_env=nil) + rails_env ||= Rails.env + config_file = File.expand_path('../../../config/resque.yml', __FILE__) + + @url = "redis://localhost:6379" + if File.exists?(config_file) + @url =YAML.load_file(config_file)[rails_env] + end + end + end +end diff --git a/lib/gitlab/redis_config.rb b/lib/gitlab/redis_config.rb deleted file mode 100644 index 4949c6db539..00000000000 --- a/lib/gitlab/redis_config.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Gitlab - class RedisConfig - attr_reader :url - - def self.url - new.url - end - - def self.redis_store_options - url = new.url - redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) - if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path - end - redis_config_hash - end - - def initialize(rails_env=nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) - - @url = "redis://localhost:6379" - if File.exists?(config_file) - @url =YAML.load_file(config_file)[rails_env] - end - end - end -end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 51e746ef923..2214f855200 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -4,18 +4,19 @@ namespace :cache do desc "GitLab | Clear redis cache" task :clear => :environment do - redis = Redis.new(url: Gitlab::RedisConfig.url) - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end end |