diff options
| -rw-r--r-- | CHANGELOG | 1 | ||||
| -rw-r--r-- | config/application.rb | 3 | ||||
| -rw-r--r-- | config/initializers/session_store.rb | 4 | ||||
| -rw-r--r-- | config/initializers/sidekiq.rb | 14 | ||||
| -rw-r--r-- | config/mail_room.yml | 53 | ||||
| -rw-r--r-- | config/resque.yml.example | 34 | ||||
| -rw-r--r-- | doc/administration/high_availability/redis.md | 301 | ||||
| -rw-r--r-- | doc/install/installation.md | 8 | ||||
| -rw-r--r-- | lib/gitlab/mail_room.rb | 47 | ||||
| -rw-r--r-- | lib/gitlab/redis.rb | 94 | ||||
| -rw-r--r-- | spec/config/mail_room_spec.rb | 43 | ||||
| -rw-r--r-- | spec/fixtures/config/redis_new_format_host.yml | 29 | ||||
| -rw-r--r-- | spec/fixtures/config/redis_new_format_socket.yml | 6 | ||||
| -rw-r--r-- | spec/fixtures/config/redis_old_format_host.yml | 5 | ||||
| -rw-r--r-- | spec/fixtures/config/redis_old_format_socket.yml | 3 | ||||
| -rw-r--r-- | spec/lib/gitlab/redis_spec.rb | 80 |
16 files changed, 604 insertions, 121 deletions
diff --git a/CHANGELOG b/CHANGELOG index 58751448d4a..845e68c62af 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -20,6 +20,7 @@ v 8.11.0 (unreleased) - Optimize maximum user access level lookup in loading of notes - Add "No one can push" as an option for protected branches. !5081 - Improve performance of AutolinkFilter#text_parse by using XPath + - Add experimental Redis Sentinel support !1877 - Environments have an url to link to - Update `timeago` plugin to use multiple string/locale settings - Remove unused images (ClemMakesApps) diff --git a/config/application.rb b/config/application.rb index 06ebb14a5fe..4a9ed41cbf8 100644 --- a/config/application.rb +++ b/config/application.rb @@ -107,7 +107,8 @@ module Gitlab end end - redis_config_hash = Gitlab::Redis.redis_store_options + # Use Redis caching across all environments + redis_config_hash = Gitlab::Redis.params redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever config.cache_store = :redis_store, redis_config_hash diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 0d9d87bac00..70be2617cab 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -13,9 +13,9 @@ end if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - redis_config = Gitlab::Redis.redis_store_options + redis_config = Gitlab::Redis.params redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE - + Gitlab::Application.config.session_store( :redis_store, # Using the cookie_store would enable session replay attacks. servers: redis_config, diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index cf49ec2194c..f7e714cd6bc 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,8 +1,9 @@ +# Custom Redis configuration +redis_config_hash = Gitlab::Redis.params +redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE + Sidekiq.configure_server do |config| - config.redis = { - url: Gitlab::Redis.url, - namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE - } + config.redis = redis_config_hash config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] @@ -39,8 +40,5 @@ Sidekiq.configure_server do |config| end Sidekiq.configure_client do |config| - config.redis = { - url: Gitlab::Redis.url, - namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE - } + config.redis = redis_config_hash end diff --git a/config/mail_room.yml b/config/mail_room.yml index 7cab24b295e..c639f8260aa 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -1,47 +1,36 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# :mailboxes: -<% -require "yaml" -require "json" -require_relative "lib/gitlab/redis" unless defined?(Gitlab::Redis) + <% + require_relative "lib/gitlab/mail_room" unless defined?(Gitlab::MailRoom) + config = Gitlab::MailRoom.config -rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development" - -config_file = ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] || "config/gitlab.yml" -if File.exists?(config_file) - all_config = YAML.load_file(config_file)[rails_env] - - config = all_config["incoming_email"] || {} - config['enabled'] = false if config['enabled'].nil? - config['port'] = 143 if config['port'].nil? - config['ssl'] = false if config['ssl'].nil? - config['start_tls'] = false if config['start_tls'].nil? - config['mailbox'] = "inbox" if config['mailbox'].nil? - - if config['enabled'] && config['address'] - redis_url = Gitlab::Redis.new(rails_env).url - %> + if Gitlab::MailRoom.enabled? + %> - - :host: <%= config['host'].to_json %> - :port: <%= config['port'].to_json %> - :ssl: <%= config['ssl'].to_json %> - :start_tls: <%= config['start_tls'].to_json %> - :email: <%= config['user'].to_json %> - :password: <%= config['password'].to_json %> + :host: <%= config[:host].to_json %> + :port: <%= config[:port].to_json %> + :ssl: <%= config[:ssl].to_json %> + :start_tls: <%= config[:start_tls].to_json %> + :email: <%= config[:user].to_json %> + :password: <%= config[:password].to_json %> + :idle_timeout: 60 - :name: <%= config['mailbox'].to_json %> + :name: <%= config[:mailbox].to_json %> :delete_after_delivery: true :delivery_method: sidekiq :delivery_options: - :redis_url: <%= redis_url.to_json %> - :namespace: resque:gitlab + :redis_url: <%= config[:redis_url].to_json %> + :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %> :queue: incoming_email :worker: EmailReceiverWorker :arbitration_method: redis :arbitration_options: - :redis_url: <%= redis_url.to_json %> - :namespace: mail_room:gitlab + :redis_url: <%= config[:redis_url].to_json %> + :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %> + <% end %> -<% end %> diff --git a/config/resque.yml.example b/config/resque.yml.example index d98f43f71b2..0c19d8bc1d3 100644 --- a/config/resque.yml.example +++ b/config/resque.yml.example @@ -1,6 +1,34 @@ # If you change this file in a Merge Request, please also create # a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests # -development: redis://localhost:6379 -test: redis://localhost:6379 -production: unix:/var/run/redis/redis.sock +development: + url: redis://localhost:6379 + # sentinels: + # - + # host: localhost + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26381 # point to sentinel, not to redis port +test: + url: redis://localhost:6379 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6379 + # sentinels: + # - + # host: slave1 + # port: 26379 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26379 # point to sentinel, not to redis port diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index f6153216f33..bc424330656 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -1,7 +1,12 @@ # Configuring Redis for GitLab HA -You can choose to install and manage Redis yourself, or you can use GitLab -Omnibus packages to help. +You can choose to install and manage Redis yourself, or you can use the one +that comes bundled with GitLab Omnibus packages. + +> **Note:** Redis does not require authentication by default. See + [Redis Security](http://redis.io/topics/security) documentation for more + information. We recommend using a combination of a Redis password and tight + firewall rules to secure your Redis service. ## Configure your own Redis server @@ -9,49 +14,293 @@ If you're hosting GitLab on a cloud provider, you can optionally use a managed service for Redis. For example, AWS offers a managed ElastiCache service that runs Redis. -> **Note:** Redis does not require authentication by default. See - [Redis Security](http://redis.io/topics/security) documentation for more - information. We recommend using a combination of a Redis password and tight - firewall rules to secure your Redis service. +## Configure Redis using Omnibus -## Configure using Omnibus +If you don't want to bother setting up your own Redis server, you can use the +one bundled with Omnibus. In this case, you should disable all services except +Redis. 1. Download/install GitLab Omnibus using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/downloads). Do not complete other steps on the download page. 1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. Be sure to change the `external_url` to match your eventual GitLab front-end - URL. + URL: ```ruby - external_url 'https://gitlab.example.com' + external_url 'https://gitlab.example.com' - # Disable all components except Redis - redis['enable'] = true - bootstrap['enable'] = false - nginx['enable'] = false - unicorn['enable'] = false - sidekiq['enable'] = false - postgresql['enable'] = false - gitlab_workhorse['enable'] = false - mailroom['enable'] = false + # Disable all services except Redis + redis['enable'] = true + bootstrap['enable'] = false + nginx['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + postgresql['enable'] = false + gitlab_workhorse['enable'] = false + mailroom['enable'] = false - # Redis configuration - redis['port'] = 6379 - redis['bind'] = '0.0.0.0' + # Redis configuration + redis['port'] = 6379 + redis['bind'] = '0.0.0.0' - # If you wish to use Redis authentication (recommended) - redis['password'] = 'Redis Password' + # If you wish to use Redis authentication (recommended) + redis['password'] = 'Redis Password' ``` 1. Run `sudo gitlab-ctl reconfigure` to install and configure PostgreSQL. > **Note**: This `reconfigure` step will result in some errors. That's OK - don't be alarmed. + 1. Run `touch /etc/gitlab/skip-auto-migrations` to prevent database migrations from running on upgrade. Only the primary GitLab application server should handle migrations. +## Experimental Redis Sentinel support + +> [Introduced][ce-1877] in GitLab 8.11. + +Since GitLab 8.11, you can configure a list of Redis Sentinel servers that +will monitor a group of Redis servers to provide you with a standard failover +support. + +There is currently one exception to the Sentinel support: `mail_room`, the +component that processes incoming emails. It doesn't support Sentinel yet, but +we hope to integrate a future release that does support it. + +To get a better understanding on how to correctly setup Sentinel, please read +the [Redis Sentinel documentation](http://redis.io/topics/sentinel) first, as +failing to configure it correctly can lead to data loss. + +The configuration consists of three parts: + +- Redis setup +- Sentinel setup +- GitLab setup + +Read carefully how to configure those components below. + +### Redis setup + +You must have at least 2 Redis servers: 1 Master, 1 or more Slaves. +They should be configured the same way and with similar server specs, as +in a failover situation, any Slave can be elected as the new Master by +the Sentinel servers. + +In a minimal setup, the only required change for the slaves in `redis.conf` +is the addition of a `slaveof` line pointing to the initial master. +You can increase the security by defining a `requirepass` configuration in +the master, and `masterauth` in slaves. + +--- + +**Configuring your own Redis server** + +1. Add to the slaves' `redis.conf`: + + ```conf + # IP and port of the master Redis server + slaveof 10.10.10.10 6379 + ``` + +1. Optionally, set up password authentication for increased security. + Add the following to master's `redis.conf`: + + ```conf + # Optional password authentication for increased security + requirepass "<password>" + ``` + +1. Then add this line to all the slave servers' `redis.conf`: + + ```conf + masterauth "<password>" + ``` + +1. Restart the Redis services for the changes to take effect. + +--- + +**Using Redis via Omnibus** + +1. Edit `/etc/gitlab/gitlab.rb` of a master Redis machine (usualy a single machine): + + ```ruby + ## Redis TCP support (will disable UNIX socket transport) + redis['bind'] = '0.0.0.0' # or specify an IP to bind to a single one + redis['port'] = 6379 + + ## Master redis instance + redis['password'] = '<huge password string here>' + ``` + +1. Edit `/etc/gitlab/gitlab.rb` of a slave Redis machine (should be one or more machines): + + ```ruby + ## Redis TCP support (will disable UNIX socket transport) + redis['bind'] = '0.0.0.0' # or specify an IP to bind to a single one + redis['port'] = 6379 + + ## Slave redis instance + redis['master_ip'] = '10.10.10.10' # IP of master Redis server + redis['master_port'] = 6379 # Port of master Redis server + redis['master_password'] = "<huge password string here>" + ``` + +1. Reconfigure the GitLab for the changes to take effect: `sudo gitlab-ctl reconfigure` + +--- + +Now that the Redis servers are all set up, let's configure the Sentinel +servers. + +### Sentinel setup + +We don't provide yet an automated way to setup and run the Sentinel daemon +from Omnibus installation method. You must follow the instructions below and +run it by yourself. + +The support for Sentinel in Ruby has some [caveats](https://github.com/redis/redis-rb/issues/531). +While you can give any name for the `master-group-name` part of the +configuration, as in this example: + +```conf +sentinel monitor <master-group-name> <ip> <port> <quorum> +``` + +,for it to work in Ruby, you have to use the "hostname" of the master Redis +server, otherwise you will get an error message like: +`Redis::CannotConnectError: No sentinels available.`. Read +[Sentinel troubleshooting](#sentinel-troubleshooting) for more information. + +Here is an example configuration file (`sentinel.conf`) for a Sentinel node: + +```conf +port 26379 +sentinel monitor master-redis.example.com 10.10.10.10 6379 1 +sentinel down-after-milliseconds master-redis.example.com 10000 +sentinel config-epoch master-redis.example.com 0 +sentinel leader-epoch master-redis.example.com 0 +``` + +--- + +The final part is to inform the main GitLab application server of the Redis +master and the new sentinels servers. + +### GitLab setup + +You can enable or disable sentinel support at any time in new or existing +installations. From the GitLab application perspective, all it requires is +the correct credentials for the master Redis and for a few Sentinel nodes. + +It doesn't require a list of all Sentinel nodes, as in case of a failure, +the application will need to query only one of them. + +>**Note:** +The following steps should be performed in the [GitLab application server](gitlab.md). + +**For source based installations** + +1. Edit `/home/git/gitlab/config/resque.yml` following the example in + `/home/git/gitlab/config/resque.yml.example`, and uncomment the sentinels + line, changing to the correct server credentials. +1. Restart GitLab for the changes to take effect. + +**For Omnibus installations** + +1. Edit `/etc/gitlab/gitlab.rb` and add/change the following lines: + + ```ruby + gitlab-rails['redis_host'] = "master-redis.example.com" + gitlab-rails['redis_port'] = 6379 + gitlab-rails['redis_password'] = '<huge password string here>' + gitlab-rails['redis_sentinels'] = [ + {'host' => '10.10.10.1', 'port' => 26379}, + {'host' => '10.10.10.2', 'port' => 26379}, + {'host' => '10.10.10.3', 'port' => 26379} + ] + ``` + +1. [Reconfigure] the GitLab for the changes to take effect. + +### Sentinel troubleshooting + +If you get an error like: `Redis::CannotConnectError: No sentinels available.`, +there may be something wrong with your configuration files or it can be related +to [this issue][gh-531] ([pull request][gh-534] that should make things better). + +It's a bit rigid the way you have to config `resque.yml` and `sentinel.conf`, +otherwise `redis-rb` will not work properly. + +The hostname ('my-primary-redis') of the primary Redis server (`sentinel.conf`) +**must** match the one configured in GitLab (`resque.yml` for source installations +or `gitlab-rails['redis_*']` in Omnibus) and it must be valid ex: + +```conf +# sentinel.conf: +sentinel monitor my-primary-redis 10.10.10.10 6379 1 +sentinel down-after-milliseconds my-primary-redis 10000 +sentinel config-epoch my-primary-redis 0 +sentinel leader-epoch my-primary-redis 0 +``` + +```yaml +# resque.yaml +production: + url: redis://my-primary-redis:6378 + sentinels: + - + host: slave1 + port: 26380 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port +``` + +When in doubt, please read [Redis Sentinel documentation](http://redis.io/topics/sentinel) + +--- + +To make sure your configuration is correct: + +1. SSH into your GitLab application server +1. Enter the Rails console: + + ``` + # For Omnibus installations + sudo gitlab-rails console + + # For source installations + sudo -u git rails console RAILS_ENV=production + ``` + +1. Run in the console: + + ```ruby + redis = Redis.new(Gitlab::Redis.params) + redis.info + ``` + + Keep this screen open and try to simulate a failover below. + +1. To simulate a failover on master Redis, SSH into the Redis server and run: + + ```bash + # port must match your master redis port + redis-cli -h localhost -p 6379 DEBUG sleep 60 + ``` + +1. Then back in the Rails console from the first step, run: + + ``` + redis.info + ``` + + You should see a different port after a few seconds delay + (the failover/reconnect time). + --- Read more on high-availability configuration: @@ -60,3 +309,9 @@ Read more on high-availability configuration: 1. [Configure NFS](nfs.md) 1. [Configure the GitLab application servers](gitlab.md) 1. [Configure the load balancers](load_balancer.md) + +[ce-1877]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/1877 +[restart]: ../restart_gitlab.md#installations-from-source +[reconfigure]: ../restart_gitlab.md#omnibus-gitlab-reconfigure +[gh-531]: https://github.com/redis/redis-rb/issues/531 +[gh-534]: https://github.com/redis/redis-rb/issues/534 diff --git a/doc/install/installation.md b/doc/install/installation.md index af8e31a705b..acf294b397d 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -588,15 +588,17 @@ for the changes to take effect. ### Custom Redis Connection -If you'd like Resque to connect to a Redis server on a non-standard port or on a different host, you can configure its connection string via the `config/resque.yml` file. +If you'd like to connect to a Redis server on a non-standard port or on a different host, you can configure its connection string via the `config/resque.yml` file. # example - production: redis://redis.example.tld:6379 + production: + url: redis://redis.example.tld:6379 If you want to connect the Redis server via socket, then use the "unix:" URL scheme and the path to the Redis socket file in the `config/resque.yml` file. # example - production: unix:/path/to/redis/socket + production: + url: unix:/path/to/redis/socket ### Custom SSH Connection diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb new file mode 100644 index 00000000000..12999a90a29 --- /dev/null +++ b/lib/gitlab/mail_room.rb @@ -0,0 +1,47 @@ +require 'yaml' +require 'json' +require_relative 'redis' unless defined?(Gitlab::Redis) + +module Gitlab + module MailRoom + class << self + def enabled? + config[:enabled] && config[:address] + end + + def config + @config ||= fetch_config + end + + def reset_config! + @config = nil + end + + private + + def fetch_config + return {} unless File.exist?(config_file) + + rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' + all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys + + config = all_config[:incoming_email] || {} + config[:enabled] = false if config[:enabled].nil? + config[:port] = 143 if config[:port].nil? + config[:ssl] = false if config[:ssl].nil? + config[:start_tls] = false if config[:start_tls].nil? + config[:mailbox] = 'inbox' if config[:mailbox].nil? + + if config[:enabled] && config[:address] + config[:redis_url] = Gitlab::Redis.new(rails_env).url + end + + config + end + + def config_file + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) + end + end + end +end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 1f92986ec9a..9376b54f43b 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -1,50 +1,94 @@ +# This file should not have any direct dependency on Rails environment +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' + module Gitlab class Redis CACHE_NAMESPACE = 'cache:gitlab' SESSION_NAMESPACE = 'session:gitlab' SIDEKIQ_NAMESPACE = 'resque:gitlab' - - attr_reader :url + MAILROOM_NAMESPACE = 'mail_room:gitlab' + DEFAULT_REDIS_URL = 'redis://localhost:6379' # To be thread-safe we must be careful when writing the class instance # variables @url and @pool. Because @pool depends on @url we need two # mutexes to prevent deadlock. - URL_MUTEX = Mutex.new + PARAMS_MUTEX = Mutex.new POOL_MUTEX = Mutex.new - private_constant :URL_MUTEX, :POOL_MUTEX + private_constant :PARAMS_MUTEX, :POOL_MUTEX - def self.url - @url || URL_MUTEX.synchronize { @url = new.url } - end + class << self + def params + @params || PARAMS_MUTEX.synchronize { @params = new.params } + end + + # @deprecated Use .params instead to get sentinel support + def url + new.url + end - def self.with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(url: url) } + def with + if @pool.nil? + POOL_MUTEX.synchronize do + @pool = ConnectionPool.new { ::Redis.new(params) } + end end + @pool.with { |redis| yield redis } end - @pool.with { |redis| yield redis } + + def reset_params! + @params = nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env end - def self.redis_store_options - url = new.url - redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) end - redis_config_hash end - def initialize(rails_env = nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) + def raw_config_hash + config_data = fetch_config - @url = "redis://localhost:6379" - if File.exist?(config_file) - @url = YAML.load_file(config_file)[rails_env] + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: DEFAULT_REDIS_URL } end end + + def fetch_config + file = config_file + File.exist?(file) ? YAML.load_file(file)[@rails_env] : false + end + + def config_file + File.expand_path('../../../config/resque.yml', __FILE__) + end end end diff --git a/spec/config/mail_room_spec.rb b/spec/config/mail_room_spec.rb index 6fad7e2b9e7..c5d3cd70acc 100644 --- a/spec/config/mail_room_spec.rb +++ b/spec/config/mail_room_spec.rb @@ -1,53 +1,48 @@ -require "spec_helper" +require 'spec_helper' -describe "mail_room.yml" do - let(:config_path) { "config/mail_room.yml" } +describe 'mail_room.yml' do + let(:config_path) { 'config/mail_room.yml' } let(:configuration) { YAML.load(ERB.new(File.read(config_path)).result) } - context "when incoming email is disabled" do + context 'when incoming email is disabled' do before do - ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = Rails.root.join("spec/fixtures/mail_room_disabled.yml").to_s + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = Rails.root.join('spec/fixtures/mail_room_disabled.yml').to_s + Gitlab::MailRoom.reset_config! end after do - ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = nil + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = nil end - it "contains no configuration" do + it 'contains no configuration' do expect(configuration[:mailboxes]).to be_nil end end - context "when incoming email is enabled" do + context 'when incoming email is enabled' do before do - ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = Rails.root.join("spec/fixtures/mail_room_enabled.yml").to_s + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = Rails.root.join('spec/fixtures/mail_room_enabled.yml').to_s + Gitlab::MailRoom.reset_config! end after do - ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = nil + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = nil end - it "contains the intended configuration" do + it 'contains the intended configuration' do expect(configuration[:mailboxes].length).to eq(1) mailbox = configuration[:mailboxes].first - expect(mailbox[:host]).to eq("imap.gmail.com") + expect(mailbox[:host]).to eq('imap.gmail.com') expect(mailbox[:port]).to eq(993) expect(mailbox[:ssl]).to eq(true) expect(mailbox[:start_tls]).to eq(false) - expect(mailbox[:email]).to eq("gitlab-incoming@gmail.com") - expect(mailbox[:password]).to eq("[REDACTED]") - expect(mailbox[:name]).to eq("inbox") - - redis_config_file = Rails.root.join('config', 'resque.yml') - - redis_url = - if File.exist?(redis_config_file) - YAML.load_file(redis_config_file)[Rails.env] - else - "redis://localhost:6379" - end + expect(mailbox[:email]).to eq('gitlab-incoming@gmail.com') + expect(mailbox[:password]).to eq('[REDACTED]') + expect(mailbox[:name]).to eq('inbox') + + redis_url = Gitlab::Redis.url expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url) expect(mailbox[:arbitration_options][:redis_url]).to eq(redis_url) diff --git a/spec/fixtures/config/redis_new_format_host.yml b/spec/fixtures/config/redis_new_format_host.yml new file mode 100644 index 00000000000..13772677a45 --- /dev/null +++ b/spec/fixtures/config/redis_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: localhost + port: 26380 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: localhost + port: 26380 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6379/99 + sentinels: + - + host: slave1 + port: 26380 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/redis_new_format_socket.yml b/spec/fixtures/config/redis_new_format_socket.yml new file mode 100644 index 00000000000..4e76830c281 --- /dev/null +++ b/spec/fixtures/config/redis_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.sock +test: + url: unix:/path/to/redis.sock +production: + url: unix:/path/to/redis.sock diff --git a/spec/fixtures/config/redis_old_format_host.yml b/spec/fixtures/config/redis_old_format_host.yml new file mode 100644 index 00000000000..253d0a994f5 --- /dev/null +++ b/spec/fixtures/config/redis_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6379/99 +test: redis://:mypassword@localhost:6379/99 +production: redis://:mypassword@localhost:6379/99 diff --git a/spec/fixtures/config/redis_old_format_socket.yml b/spec/fixtures/config/redis_old_format_socket.yml new file mode 100644 index 00000000000..fd31ce8ea3d --- /dev/null +++ b/spec/fixtures/config/redis_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.sock +test: unix:/path/to/old/redis.sock +production: unix:/path/to/old/redis.sock diff --git a/spec/lib/gitlab/redis_spec.rb b/spec/lib/gitlab/redis_spec.rb new file mode 100644 index 00000000000..879ed30841c --- /dev/null +++ b/spec/lib/gitlab/redis_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' + +describe Gitlab::Redis do + let(:redis_config) { Rails.root.join('config', 'resque.yml').to_s } + + before(:each) { described_class.reset_params! } + after(:each) { described_class.reset_params! } + + describe '.params' do + subject { described_class.params } + + context 'when url contains unix socket reference' do + let(:config_old) { Rails.root.join('spec/fixtures/config/redis_old_format_socket.yml').to_s } + let(:config_new) { Rails.root.join('spec/fixtures/config/redis_new_format_socket.yml').to_s } + + context 'with old format' do + it 'returns path key instead' do + expect_any_instance_of(described_class).to receive(:config_file) { config_old } + + is_expected.to include(path: '/path/to/old/redis.sock') + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + it 'returns path key instead' do + expect_any_instance_of(described_class).to receive(:config_file) { config_new } + + is_expected.to include(path: '/path/to/redis.sock') + is_expected.not_to have_key(:url) + end + end + end + + context 'when url is host based' do + let(:config_old) { Rails.root.join('spec/fixtures/config/redis_old_format_host.yml') } + let(:config_new) { Rails.root.join('spec/fixtures/config/redis_new_format_host.yml') } + + context 'with old format' do + it 'returns hash with host, port, db, and password' do + expect_any_instance_of(described_class).to receive(:config_file) { config_old } + + is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99) + is_expected.not_to have_key(:url) + end + end + + context 'with new format' do + it 'returns hash with host, port, db, and password' do + expect_any_instance_of(described_class).to receive(:config_file) { config_new } + + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99) + is_expected.not_to have_key(:url) + end + end + end + end + + describe '#raw_config_hash' do + it 'returns default redis url when no config file is present' do + expect(subject).to receive(:fetch_config) { false } + + expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::DEFAULT_REDIS_URL) + end + + it 'returns old-style single url config in a hash' do + expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' } + expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379') + end + + end + + describe '#fetch_config' do + it 'returns false when no config file is present' do + allow(File).to receive(:exist?).with(redis_config) { false } + + expect(subject.send(:fetch_config)).to be_falsey + end + end +end |