diff options
| author | Paul Charlton <techguru@byiq.com> | 2017-07-11 03:35:47 +0000 |
|---|---|---|
| committer | Robert Speicher <robert@gitlab.com> | 2017-07-11 03:35:47 +0000 |
| commit | cb3b4a15e6913bc28ee2ecaab017a4c3f08c438e (patch) | |
| tree | dc3915aa94508d76df2480a8e26ec4b33797a320 | |
| parent | 4daa6da5407d235cbe4f7a787eaa29304446a870 (diff) | |
| download | gitlab-ce-cb3b4a15e6913bc28ee2ecaab017a4c3f08c438e.tar.gz | |
Support multiple Redis instances based on queue type
118 files changed, 1158 insertions, 450 deletions
diff --git a/.flayignore b/.flayignore index 47597025115..e2d0a2e50c5 100644 --- a/.flayignore +++ b/.flayignore @@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb lib/gitlab/diff/position_tracer.rb app/policies/project_policy.rb app/models/concerns/relative_positioning.rb +lib/gitlab/redis/*.rb diff --git a/.gitignore b/.gitignore index 0d6194dd1e5..3baf640a9c3 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ eslint-report.html /config/initializers/smtp_settings.rb /config/initializers/relative_url.rb /config/resque.yml +/config/redis.cache.yml +/config/redis.queues.yml +/config/redis.shared_state.yml /config/unicorn.rb /config/secrets.yml /config/sidekiq.yml diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index abc832e6ddc..3dbacbbc897 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -4,7 +4,10 @@ class HealthController < ActionController::Base CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::RedisCheck, + Gitlab::HealthChecks::Redis::RedisCheck, + Gitlab::HealthChecks::Redis::CacheCheck, + Gitlab::HealthChecks::Redis::QueuesCheck, + Gitlab::HealthChecks::Redis::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck ].freeze diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index fb0fbb43fb1..c6d23898560 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -149,7 +149,7 @@ module Ci private def cleanup_runner_queue - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| redis.del(runner_queue_key) end end diff --git a/app/models/project.rb b/app/models/project.rb index d5760164663..e50818e3dff 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1386,15 +1386,15 @@ class Project < ActiveRecord::Base end def pushes_since_gc - Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i } + Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) } + Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) } end def reset_pushes_since_gc - Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) } + Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) } end def route_map_for(commit_sha) @@ -1457,7 +1457,7 @@ class Project < ActiveRecord::Base from && self != from end - def pushes_since_gc_redis_key + def pushes_since_gc_redis_shared_state_key "projects/#{id}/pushes_since_gc" end diff --git a/app/services/metrics_service.rb b/app/services/metrics_service.rb index d726db4e99b..c92f070601c 100644 --- a/app/services/metrics_service.rb +++ b/app/services/metrics_service.rb @@ -3,7 +3,10 @@ require 'prometheus/client/formats/text' class MetricsService CHECKS = [ Gitlab::HealthChecks::DbCheck, - Gitlab::HealthChecks::RedisCheck, + Gitlab::HealthChecks::Redis::RedisCheck, + Gitlab::HealthChecks::Redis::CacheCheck, + Gitlab::HealthChecks::Redis::QueuesCheck, + Gitlab::HealthChecks::Redis::SharedStateCheck, Gitlab::HealthChecks::FsShardsCheck ].freeze diff --git a/app/services/milestones/destroy_service.rb b/app/services/milestones/destroy_service.rb index 600ebcfbecb..e457212508f 100644 --- a/app/services/milestones/destroy_service.rb +++ b/app/services/milestones/destroy_service.rb @@ -1,17 +1,15 @@ module Milestones class DestroyService < Milestones::BaseService def execute(milestone) - return unless milestone.is_project_milestone? - Milestone.transaction do update_params = { milestone: nil } milestone.issues.each do |issue| - Issues::UpdateService.new(parent, current_user, update_params).execute(issue) + Issues::UpdateService.new(project, current_user, update_params).execute(issue) end milestone.merge_requests.each do |merge_request| - MergeRequests::UpdateService.new(parent, current_user, update_params).execute(merge_request) + MergeRequests::UpdateService.new(project, current_user, update_params).execute(merge_request) end event_service.destroy_milestone(milestone, current_user) diff --git a/config/README.md b/config/README.md index 0a5ea2424e0..2778d0d4f02 100644 --- a/config/README.md +++ b/config/README.md @@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration. This file is called `resque.yml` for historical reasons. We are **NOT** using Resque at the moment. It is used to specify Redis configuration -values instead. +values when a single database instance of Redis is desired. + +# Advanced Redis configuration files + +In more advanced configurations of Redis key-value storage, it is desirable +to separate the keys by lifecycle and intended use to ease provisioning and +management of scalable Redis clusters. + +These settings provide routing and other configuration data (such as sentinel, +persistence policies, and other Redis customization) for connections +to Redis single instances, Redis sentinel, and Redis clusters. + +If desired, the routing URL provided by these settings can be used with: +1. Unix Socket + 1. named socket for each Redis instance desired. + 2. `database number` for each Redis instance desired. +2. TCP Socket + 1. `host name` or IP for each Redis instance desired + 2. TCP port number for each Redis instance desired + 3. `database number` for each Redis instance desired + +## Example URL attribute formats for GitLab Redis `.yml` configuration files +* Unix Socket, default Redis database (0) + * `url: unix:/path/to/redis.sock` + * `url: unix:/path/to/redis.sock?db=` +* Unix Socket, Redis database 44 + * `url: unix:/path/to/redis.sock?db=44` + * `url: unix:/path/to/redis.sock?extra=foo&db=44` +* TCP Socket for Redis on localhost, port 6379, database 33 + * `url: redis://:mynewpassword@localhost:6379/33` +* TCP Socket for Redis on remote host `myserver`, port 6379, database 33 + * `url: redis://:mynewpassword@myserver:6379/33` + +## redis.cache.yml + +If configured, `redis.cache.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for `Rails.cache` and other volatile non-persistent data which enhances +the performance of GitLab. +Settings here can be overridden by the environment variable +`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides +an alternate location for configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `cache` is: +1. URL from a configuration file pointed to by the +`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable +2. URL from `redis.cache.yml` +3. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. URL from `resque.yml` +5. `redis://localhost:6380` + +The order of precedence for all other configuration settings for `cache` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable +2. the configuration file `redis.cache.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + +## redis.queues.yml + +If configured, `redis.queues.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for clients of `::Gitlab::Redis::Queues`. +These queues are intended to be the foundation +of reliable inter-process communication between modules, whether on the same +host node, or within a cluster. The primary clients of the queues are +SideKiq, Mailroom, CI Runner, Workhorse, and push services. Settings here can +be overridden by the environment variable +`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for +configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `queues` is: +1. URL from a configuration file pointed to by the +`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable +2. URL from `redis.queues.yml` +3. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. URL from `resque.yml` +5. `redis://localhost:6381` + +The order of precedence for all other configuration settings for `queues` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable +2. the configuration file `redis.queues.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + +## redis.shared_state.yml + +If configured, `redis.shared_state.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for clients of `::Gitlab::Redis::SharedState` such as session state, +and rate limiting. +Settings here can be overridden by the environment variable +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides +an alternate location for configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `shared_state` is: +1. URL from a configuration file pointed to by the +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable +2. URL from `redis.shared_state.yml` +3. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. URL from `resque.yml` +5. `redis://localhost:6382` + +The order of precedence for all other configuration settings for `shared_state` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable +2. the configuration file `redis.shared_state.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + diff --git a/config/application.rb b/config/application.rb index 2f4e2624195..1c13cc81270 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,7 +6,9 @@ Bundler.require(:default, Rails.env) module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/redis') + require_dependency Rails.root.join('lib/gitlab/redis/cache') + require_dependency Rails.root.join('lib/gitlab/redis/queues') + require_dependency Rails.root.join('lib/gitlab/redis/shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') # Settings in config/environments/* take precedence over those specified here. @@ -142,15 +144,15 @@ module Gitlab end end - # Use Redis caching across all environments - redis_config_hash = Gitlab::Redis.params - redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE - redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever + # Use caching across all environments + caching_config_hash = Gitlab::Redis::Cache.params + caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE + caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context - redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 - redis_config_hash[:pool_timeout] = 1 + caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 + caching_config_hash[:pool_timeout] = 1 end - config.cache_store = :redis_store, redis_config_hash + config.cache_store = :redis_store, caching_config_hash config.active_record.raise_in_transactional_callbacks = true diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index ae2ca258df1..af4967521b8 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,3 +1,8 @@ -# Make sure we initialize a Redis connection pool before Sidekiq starts -# multi-threaded execution. -Gitlab::Redis.with { nil } +# Make sure we initialize a Redis connection pool before multi-threaded +# execution starts by +# 1. Sidekiq +# 2. Rails.cache +# 3. HTTP clients +Gitlab::Redis::Cache.with { nil } +Gitlab::Redis::Queues.with { nil } +Gitlab::Redis::SharedState.with { nil } diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index 65432caac2a..da8282ec924 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -1,4 +1,4 @@ -Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) } +Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) } Peek.into Peek::Views::Host Peek.into Peek::Views::PerformanceBar diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 8919f7640fe..e8213ac8ba4 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -19,12 +19,12 @@ cookie_key = if Rails.env.development? if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - redis_config = Gitlab::Redis.params - redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE + sessions_config = Gitlab::Redis::SharedState.params + sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE Gitlab::Application.config.session_store( :redis_store, # Using the cookie_store would enable session replay attacks. - servers: redis_config, + servers: sessions_config, key: cookie_key, secure: Gitlab.config.gitlab.https, httponly: true, diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 3be4cd797aa..a1cc9655319 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,12 +1,12 @@ -# Custom Redis configuration -redis_config_hash = Gitlab::Redis.params -redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE +# Custom Queues configuration +queues_config_hash = Gitlab::Redis::Queues.params +queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE # Default is to retry 25 times with exponential backoff. That's too much. Sidekiq.default_worker_options = { retry: 3 } Sidekiq.configure_server do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] @@ -54,7 +54,7 @@ Sidekiq.configure_server do |config| end Sidekiq.configure_client do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.client_middleware do |chain| chain.add Gitlab::SidekiqStatus::ClientMiddleware diff --git a/config/mail_room.yml b/config/mail_room.yml index 88d93d4bc6b..c3a5be8d38c 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -21,7 +21,7 @@ :delivery_method: sidekiq :delivery_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %> :queue: email_receiver :worker: EmailReceiverWorker <% if config[:sentinels] %> @@ -36,7 +36,7 @@ :arbitration_method: redis :arbitration_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %> <% if config[:sentinels] %> :sentinels: <% config[:sentinels].each do |sentinel| %> diff --git a/config/redis.cache.yml.example b/config/redis.cache.yml.example new file mode 100644 index 00000000000..27478f0a93e --- /dev/null +++ b/config/redis.cache.yml.example @@ -0,0 +1,38 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/10 + # + # url: redis://localhost:6380 + # sentinels: + # - + # host: localhost + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26380 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/10 + # + # url: redis://localhost:6380 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.cache.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6380 + # sentinels: + # - + # host: slave1 + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26380 # point to sentinel, not to redis port diff --git a/config/redis.queues.yml.example b/config/redis.queues.yml.example new file mode 100644 index 00000000000..dab1f26b096 --- /dev/null +++ b/config/redis.queues.yml.example @@ -0,0 +1,38 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/11 + # + # url: redis://localhost:6381 + # sentinels: + # - + # host: localhost + # port: 26381 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26381 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/11 + # + # url: redis://localhost:6381 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.queues.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6381 + # sentinels: + # - + # host: slave1 + # port: 26381 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26381 # point to sentinel, not to redis port diff --git a/config/redis.shared_state.yml.example b/config/redis.shared_state.yml.example new file mode 100644 index 00000000000..9371e3619b7 --- /dev/null +++ b/config/redis.shared_state.yml.example @@ -0,0 +1,38 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/12 + # + # url: redis://localhost:6382 + # sentinels: + # - + # host: localhost + # port: 26382 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26382 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/12 + # + # url: redis://localhost:6382 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.shared_state.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6382 + # sentinels: + # - + # host: slave1 + # port: 26382 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26382 # point to sentinel, not to redis port diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb index 397a9a2d28e..cb1b4f1855d 100644 --- a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb +++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb @@ -56,7 +56,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration end def activities(from, to, page: 1) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i, with_scores: true, limit: limit(page)) @@ -64,7 +64,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration end def activities_count(from, to) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i) end end diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md index fe982ea83c2..8b7a515a076 100644 --- a/doc/administration/high_availability/redis_source.md +++ b/doc/administration/high_availability/redis_source.md @@ -4,6 +4,11 @@ This is the documentation for configuring a Highly Available Redis setup when you have installed Redis all by yourself and not using the bundled one that comes with the Omnibus packages. +Note also that you may elect to override all references to +`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis +settings outlined in +[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md). + We cannot stress enough the importance of reading the [Overview section](redis.md#overview) of the Omnibus Redis HA as it provides some invaluable information to the configuration of Redis. Please proceed to diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index 93521e976d5..3a35aff8366 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -15,6 +15,12 @@ prefixed with 'session:gitlab:', so they would look like 'session:gitlab:976aa289e2189b17d7ef525a6702ace9'. Below we describe how to remove the keys in the old format. +**Note:** the instructions below must be modified in accordance with your +configuration settings if you have used the advanced Redis +settings outlined in +[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md). + + First we define a shell function with the proper Redis connection details. ``` diff --git a/doc/install/installation.md b/doc/install/installation.md index dfa25deb961..992ff162efb 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -420,12 +420,6 @@ GitLab Shell is an SSH access and repository management software developed speci **Note:** Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in /etc/hosts ("127.0.0.1 hostname"). This might be necessary for example if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check will fail with "Check GitLab API access: FAILED. code: 401" and pushing commits will be rejected with "[remote rejected] master -> master (hook declined)". -**Note:** GitLab Shell application startup time can be greatly reduced by disabling RubyGems. This can be done in several manners: - -* Export `RUBYOPT=--disable-gems` environment variable for the processes -* Compile Ruby with `configure --disable-rubygems` to disable RubyGems by default. Not recommened for system-wide Ruby. -* Omnibus GitLab [replaces the *shebang* line of the `gitlab-shell/bin/*` scripts](https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/1707) - ### Install gitlab-workhorse GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150..baa1f802d8a 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ module Gitlab time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9c2e09943b0..dba37892863 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ module Gitlab end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ module Gitlab end def load_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d..e63e5437331 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ module Gitlab end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ module Gitlab Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7fa02f3d7b3..791a3c36476 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab def cached_application_settings begin ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL + rescue ::Redis::BaseError, ::Errno::ENOENT # In case Redis isn't running or the Redis UNIX socket file is not available end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 33f8939bc61..1a697396ff1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -145,7 +145,7 @@ module Gitlab def track_rename(type, old_path, new_path) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.lpush(key, [old_path, new_path].to_json) redis.expire(key, 2.weeks.to_i) end @@ -155,7 +155,7 @@ module Gitlab def reverts_for_type(type) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| failed_reverts = [] while rename_info = redis.lpop(key) diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 072fcfc65e6..21172ff8d93 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,10 +24,10 @@ module Gitlab SecureRandom.hex end - def redis_key(key) + def redis_shared_state_key(key) raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) - "#{REDIS_NAMESPACE}#{key}" + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index a0f46594eb1..3784f6c4947 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -26,17 +26,17 @@ module Gitlab EOS def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -45,24 +45,24 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis.with do |redis| - result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout]) + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb new file mode 100644 index 00000000000..a28658d42d4 --- /dev/null +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class CacheCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_cache_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Cache.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 00000000000..f97d50d3947 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 00000000000..fe4e3c4a3ab --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 00000000000..e3244392902 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb deleted file mode 100644 index 57bbe5b3ad0..00000000000 --- a/lib/gitlab/health_checks/redis_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a..8e57ba831c5 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ module Gitlab end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ module Gitlab private - def redis_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8..9f432673a6e 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,11 +34,11 @@ module Gitlab config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb deleted file mode 100644 index bc5370de32a..00000000000 --- a/lib/gitlab/redis.rb +++ /dev/null @@ -1,102 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - if Sidekiq.server? - # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: DEFAULT_REDIS_URL } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb new file mode 100644 index 00000000000..b0da516ff83 --- /dev/null +++ b/lib/gitlab/redis/cache.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Cache < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 00000000000..f9249d05565 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,35 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 00000000000..395dcf082da --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 00000000000..10c16a962ca --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,123 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze + end + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = ERB.new(File.read(config_file_name)).result.freeze + rescue Errno::ENOENT + @_raw_config = false + end + + @_raw_config + end + + def default_url + DEFAULT_REDIS_URL + end + + def config_file_name + # if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: self.class.default_url } + end + end + + def fetch_config + self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false + end + end + end +end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fded..125488536e1 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ module Gitlab BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ module Gitlab cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 4aef23b6aee..ee4f2b773ea 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -176,7 +176,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 125a3d560d6..564aa141952 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -5,12 +5,12 @@ namespace :cache do desc "GitLab | Clear redis cache" task redis: :environment do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| cursor = REDIS_SCAN_START_STOP loop do cursor, keys = redis.scan( cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", count: REDIS_CLEAR_BATCH_SIZE ) diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 03de59f27ad..68114d149c4 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -12,9 +12,6 @@ fi # gems could not be found under some circumstance. No idea why, hours wasted. retry gem install knapsack fog-aws mime-types -cp config/resque.yml.example config/resque.yml -sed -i 's/localhost/redis/g' config/resque.yml - cp config/gitlab.yml.example config/gitlab.yml # Determine the database by looking at the job name. @@ -37,6 +34,18 @@ else # Assume it's mysql sed -i 's/# host:.*/host: mysql/g' config/database.yml fi +cp config/resque.yml.example config/resque.yml +sed -i 's/localhost/redis/g' config/resque.yml + +cp config/redis.cache.yml.example config/redis.cache.yml +sed -i 's/localhost/redis/g' config/redis.cache.yml + +cp config/redis.queues.yml.example config/redis.queues.yml +sed -i 's/localhost/redis/g' config/redis.queues.yml + +cp config/redis.shared_state.yml.example config/redis.shared_state.yml +sed -i 's/localhost/redis/g' config/redis.shared_state.yml + if [ "$SETUP_DB" != "false" ]; then bundle exec rake db:drop db:create db:schema:load db:migrate diff --git a/spec/config/mail_room_spec.rb b/spec/config/mail_room_spec.rb index 092048a6259..a31e44fa928 100644 --- a/spec/config/mail_room_spec.rb +++ b/spec/config/mail_room_spec.rb @@ -5,12 +5,12 @@ describe 'mail_room.yml' do let(:mailroom_config_path) { 'config/mail_room.yml' } let(:gitlab_config_path) { 'config/mail_room.yml' } - let(:redis_config_path) { 'config/resque.yml' } + let(:queues_config_path) { 'config/redis.queues.yml' } let(:configuration) do vars = { 'MAIL_ROOM_GITLAB_CONFIG_FILE' => absolute_path(gitlab_config_path), - 'GITLAB_REDIS_CONFIG_FILE' => absolute_path(redis_config_path) + 'GITLAB_REDIS_QUEUES_CONFIG_FILE' => absolute_path(queues_config_path) } cmd = "puts ERB.new(File.read(#{absolute_path(mailroom_config_path).inspect})).result" @@ -21,12 +21,12 @@ describe 'mail_room.yml' do end before(:each) do - stub_env('GITLAB_REDIS_CONFIG_FILE', absolute_path(redis_config_path)) - clear_redis_raw_config + stub_env('GITLAB_REDIS_QUEUES_CONFIG_FILE', absolute_path(queues_config_path)) + clear_queues_raw_config end after(:each) do - clear_redis_raw_config + clear_queues_raw_config end context 'when incoming email is disabled' do @@ -39,9 +39,9 @@ describe 'mail_room.yml' do context 'when incoming email is enabled' do let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' } - let(:redis_config_path) { 'spec/fixtures/config/redis_new_format_host.yml' } + let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' } - let(:gitlab_redis) { Gitlab::Redis.new(Rails.env) } + let(:gitlab_redis_queues) { Gitlab::Redis::Queues.new(Rails.env) } it 'contains the intended configuration' do expect(configuration[:mailboxes].length).to eq(1) @@ -56,8 +56,8 @@ describe 'mail_room.yml' do expect(mailbox[:name]).to eq('inbox') expect(mailbox[:idle_timeout]).to eq(60) - redis_url = gitlab_redis.url - sentinels = gitlab_redis.sentinels + redis_url = gitlab_redis_queues.url + sentinels = gitlab_redis_queues.sentinels expect(mailbox[:delivery_options][:redis_url]).to be_present expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url) @@ -73,8 +73,8 @@ describe 'mail_room.yml' do end end - def clear_redis_raw_config - Gitlab::Redis.remove_instance_variable(:@_raw_config) + def clear_queues_raw_config + Gitlab::Redis::Queues.remove_instance_variable(:@_raw_config) rescue NameError # raised if @_raw_config was not set; ignore end diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb index e7c19b47a6a..241c36620d1 100644 --- a/spec/controllers/health_controller_spec.rb +++ b/spec/controllers/health_controller_spec.rb @@ -19,7 +19,9 @@ describe HealthController do it 'returns proper response' do get :readiness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_check']['status']).to eq('ok') + expect(json_response['cache_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['labels']['shard']).to eq('default') end @@ -42,7 +44,9 @@ describe HealthController do it 'returns proper response' do get :liveness expect(json_response['db_check']['status']).to eq('ok') - expect(json_response['redis_check']['status']).to eq('ok') + expect(json_response['cache_check']['status']).to eq('ok') + expect(json_response['queues_check']['status']).to eq('ok') + expect(json_response['shared_state_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok') end end diff --git a/spec/controllers/metrics_controller_spec.rb b/spec/controllers/metrics_controller_spec.rb index 044c9f179ed..c46dd92b454 100644 --- a/spec/controllers/metrics_controller_spec.rb +++ b/spec/controllers/metrics_controller_spec.rb @@ -35,6 +35,30 @@ describe MetricsController do expect(response.body).to match(/^redis_ping_latency [0-9\.]+$/) end + it 'returns Caching ping metrics' do + get :index + + expect(response.body).to match(/^redis_cache_ping_timeout 0$/) + expect(response.body).to match(/^redis_cache_ping_success 1$/) + expect(response.body).to match(/^redis_cache_ping_latency [0-9\.]+$/) + end + + it 'returns Queues ping metrics' do + get :index + + expect(response.body).to match(/^redis_queues_ping_timeout 0$/) + expect(response.body).to match(/^redis_queues_ping_success 1$/) + expect(response.body).to match(/^redis_queues_ping_latency [0-9\.]+$/) + end + + it 'returns SharedState ping metrics' do + get :index + + expect(response.body).to match(/^redis_shared_state_ping_timeout 0$/) + expect(response.body).to match(/^redis_shared_state_ping_success 1$/) + expect(response.body).to match(/^redis_shared_state_ping_latency [0-9\.]+$/) + end + it 'returns file system check metrics' do get :index diff --git a/spec/controllers/sessions_controller_spec.rb b/spec/controllers/sessions_controller_spec.rb index bf922260b2f..2b4e8723b48 100644 --- a/spec/controllers/sessions_controller_spec.rb +++ b/spec/controllers/sessions_controller_spec.rb @@ -47,7 +47,7 @@ describe SessionsController do end end - context 'when using valid password', :redis do + context 'when using valid password', :clean_gitlab_redis_shared_state do include UserActivitiesHelpers let(:user) { create(:user) } diff --git a/spec/features/dashboard/issuables_counter_spec.rb b/spec/features/dashboard/issuables_counter_spec.rb index 285724f4b48..6b666934563 100644 --- a/spec/features/dashboard/issuables_counter_spec.rb +++ b/spec/features/dashboard/issuables_counter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe 'Navigation bar counter', feature: true, caching: true do +describe 'Navigation bar counter', :use_clean_rails_memory_store_caching, feature: true do let(:user) { create(:user) } let(:project) { create(:empty_project, namespace: user.namespace) } let(:issue) { create(:issue, project: project) } diff --git a/spec/features/dashboard/projects_spec.rb b/spec/features/dashboard/projects_spec.rb index 7ca002fc821..bdba22fe9a9 100644 --- a/spec/features/dashboard/projects_spec.rb +++ b/spec/features/dashboard/projects_spec.rb @@ -61,7 +61,7 @@ feature 'Dashboard Projects' do end end - describe 'with a pipeline', redis: true do + describe "with a pipeline", clean_gitlab_redis_shared_state: true do let(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit.sha) } before do diff --git a/spec/features/groups/members/sort_members_spec.rb b/spec/features/groups/members/sort_members_spec.rb index 169827f5d0d..92ff45e0cdc 100644 --- a/spec/features/groups/members/sort_members_spec.rb +++ b/spec/features/groups/members/sort_members_spec.rb @@ -68,7 +68,7 @@ feature 'Groups > Members > Sort members', feature: true do expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :redis do + scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(owner.name) @@ -76,7 +76,7 @@ feature 'Groups > Members > Sort members', feature: true do expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :redis do + scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/features/login_spec.rb b/spec/features/login_spec.rb index a8055b21cee..2a2213b67ed 100644 --- a/spec/features/login_spec.rb +++ b/spec/features/login_spec.rb @@ -41,7 +41,7 @@ feature 'Login', feature: true do expect(page).to have_content('Your account has been blocked.') end - it 'does not update Devise trackable attributes', :redis do + it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do user = create(:user, :blocked) expect { gitlab_sign_in(user) }.not_to change { user.reload.sign_in_count } @@ -55,7 +55,7 @@ feature 'Login', feature: true do expect(page).to have_content('Invalid Login or password.') end - it 'does not update Devise trackable attributes', :redis do + it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do expect { gitlab_sign_in(User.ghost) }.not_to change { User.ghost.reload.sign_in_count } end end diff --git a/spec/features/projects/members/sorting_spec.rb b/spec/features/projects/members/sorting_spec.rb index afb613f034e..dc7236fa120 100644 --- a/spec/features/projects/members/sorting_spec.rb +++ b/spec/features/projects/members/sorting_spec.rb @@ -67,7 +67,7 @@ feature 'Projects > Members > Sorting', feature: true do expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') end - scenario 'sorts by recent sign in', :redis do + scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :recent_sign_in) expect(first_member).to include(master.name) @@ -75,7 +75,7 @@ feature 'Projects > Members > Sorting', feature: true do expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') end - scenario 'sorts by oldest sign in', :redis do + scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do visit_members_list(sort: :oldest_sign_in) expect(first_member).to include(developer.name) diff --git a/spec/fixtures/config/redis_cache_config_with_env.yml b/spec/fixtures/config/redis_cache_config_with_env.yml new file mode 100644 index 00000000000..52fd5a06460 --- /dev/null +++ b/spec/fixtures/config/redis_cache_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_CACHE_URL'] %> diff --git a/spec/fixtures/config/redis_cache_new_format_host.yml b/spec/fixtures/config/redis_cache_new_format_host.yml new file mode 100644 index 00000000000..a24f3716391 --- /dev/null +++ b/spec/fixtures/config/redis_cache_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6380/10 + sentinels: + - + host: localhost + port: 26380 # point to sentinel, not to redis port + - + host: slave2 + port: 26380 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6380/10 + sentinels: + - + host: localhost + port: 26380 # point to sentinel, not to redis port + - + host: slave2 + port: 26380 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6380/10 + sentinels: + - + host: slave1 + port: 26380 # point to sentinel, not to redis port + - + host: slave2 + port: 26380 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/redis_cache_new_format_socket.yml b/spec/fixtures/config/redis_cache_new_format_socket.yml new file mode 100644 index 00000000000..3634c550163 --- /dev/null +++ b/spec/fixtures/config/redis_cache_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.cache.sock +test: + url: unix:/path/to/redis.cache.sock +production: + url: unix:/path/to/redis.cache.sock diff --git a/spec/fixtures/config/redis_cache_old_format_host.yml b/spec/fixtures/config/redis_cache_old_format_host.yml new file mode 100644 index 00000000000..3609dcd022e --- /dev/null +++ b/spec/fixtures/config/redis_cache_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6380/10 +test: redis://:mypassword@localhost:6380/10 +production: redis://:mypassword@localhost:6380/10 diff --git a/spec/fixtures/config/redis_cache_old_format_socket.yml b/spec/fixtures/config/redis_cache_old_format_socket.yml new file mode 100644 index 00000000000..26fa0eda245 --- /dev/null +++ b/spec/fixtures/config/redis_cache_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.cache.sock +test: unix:/path/to/old/redis.cache.sock +production: unix:/path/to/old/redis.cache.sock diff --git a/spec/fixtures/config/redis_new_format_host.yml b/spec/fixtures/config/redis_new_format_host.yml index 13772677a45..8d134d467e9 100644 --- a/spec/fixtures/config/redis_new_format_host.yml +++ b/spec/fixtures/config/redis_new_format_host.yml @@ -5,25 +5,25 @@ development: sentinels: - host: localhost - port: 26380 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: slave2 - port: 26381 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port test: url: redis://:mynewpassword@localhost:6379/99 sentinels: - host: localhost - port: 26380 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: slave2 - port: 26381 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port production: url: redis://:mynewpassword@localhost:6379/99 sentinels: - host: slave1 - port: 26380 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port - host: slave2 - port: 26381 # point to sentinel, not to redis port + port: 26379 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/redis_queues_config_with_env.yml b/spec/fixtures/config/redis_queues_config_with_env.yml new file mode 100644 index 00000000000..d16a9d8a7f8 --- /dev/null +++ b/spec/fixtures/config/redis_queues_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_QUEUES_URL'] %> diff --git a/spec/fixtures/config/redis_queues_new_format_host.yml b/spec/fixtures/config/redis_queues_new_format_host.yml new file mode 100644 index 00000000000..1535584d779 --- /dev/null +++ b/spec/fixtures/config/redis_queues_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6381/11 + sentinels: + - + host: localhost + port: 26381 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6381/11 + sentinels: + - + host: localhost + port: 26381 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6381/11 + sentinels: + - + host: slave1 + port: 26381 # point to sentinel, not to redis port + - + host: slave2 + port: 26381 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/redis_queues_new_format_socket.yml b/spec/fixtures/config/redis_queues_new_format_socket.yml new file mode 100644 index 00000000000..b488d84d022 --- /dev/null +++ b/spec/fixtures/config/redis_queues_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.queues.sock +test: + url: unix:/path/to/redis.queues.sock +production: + url: unix:/path/to/redis.queues.sock diff --git a/spec/fixtures/config/redis_queues_old_format_host.yml b/spec/fixtures/config/redis_queues_old_format_host.yml new file mode 100644 index 00000000000..6531748a8d7 --- /dev/null +++ b/spec/fixtures/config/redis_queues_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6381/11 +test: redis://:mypassword@localhost:6381/11 +production: redis://:mypassword@localhost:6381/11 diff --git a/spec/fixtures/config/redis_queues_old_format_socket.yml b/spec/fixtures/config/redis_queues_old_format_socket.yml new file mode 100644 index 00000000000..53f5db72758 --- /dev/null +++ b/spec/fixtures/config/redis_queues_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.queues.sock +test: unix:/path/to/old/redis.queues.sock +production: unix:/path/to/old/redis.queues.sock diff --git a/spec/fixtures/config/redis_shared_state_config_with_env.yml b/spec/fixtures/config/redis_shared_state_config_with_env.yml new file mode 100644 index 00000000000..eab7203d0de --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_config_with_env.yml @@ -0,0 +1,2 @@ +test: + url: <%= ENV['TEST_GITLAB_REDIS_SHARED_STATE_URL'] %> diff --git a/spec/fixtures/config/redis_shared_state_new_format_host.yml b/spec/fixtures/config/redis_shared_state_new_format_host.yml new file mode 100644 index 00000000000..1180b2b4a82 --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_new_format_host.yml @@ -0,0 +1,29 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: + url: redis://:mynewpassword@localhost:6382/12 + sentinels: + - + host: localhost + port: 26382 # point to sentinel, not to redis port + - + host: slave2 + port: 26382 # point to sentinel, not to redis port +test: + url: redis://:mynewpassword@localhost:6382/12 + sentinels: + - + host: localhost + port: 26382 # point to sentinel, not to redis port + - + host: slave2 + port: 26382 # point to sentinel, not to redis port +production: + url: redis://:mynewpassword@localhost:6382/12 + sentinels: + - + host: slave1 + port: 26382 # point to sentinel, not to redis port + - + host: slave2 + port: 26382 # point to sentinel, not to redis port diff --git a/spec/fixtures/config/redis_shared_state_new_format_socket.yml b/spec/fixtures/config/redis_shared_state_new_format_socket.yml new file mode 100644 index 00000000000..1b0e699729e --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_new_format_socket.yml @@ -0,0 +1,6 @@ +development: + url: unix:/path/to/redis.shared_state.sock +test: + url: unix:/path/to/redis.shared_state.sock +production: + url: unix:/path/to/redis.shared_state.sock diff --git a/spec/fixtures/config/redis_shared_state_old_format_host.yml b/spec/fixtures/config/redis_shared_state_old_format_host.yml new file mode 100644 index 00000000000..fef5e768c5d --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_old_format_host.yml @@ -0,0 +1,5 @@ +# redis://[:password@]host[:port][/db-number][?option=value] +# more details: http://www.iana.org/assignments/uri-schemes/prov/redis +development: redis://:mypassword@localhost:6382/12 +test: redis://:mypassword@localhost:6382/12 +production: redis://:mypassword@localhost:6382/12 diff --git a/spec/fixtures/config/redis_shared_state_old_format_socket.yml b/spec/fixtures/config/redis_shared_state_old_format_socket.yml new file mode 100644 index 00000000000..4746afbb5ef --- /dev/null +++ b/spec/fixtures/config/redis_shared_state_old_format_socket.yml @@ -0,0 +1,3 @@ +development: unix:/path/to/old/redis.shared_state.sock +test: unix:/path/to/old/redis.shared_state.sock +production: unix:/path/to/old/redis.shared_state.sock diff --git a/spec/helpers/issuables_helper_spec.rb b/spec/helpers/issuables_helper_spec.rb index d2e918ef014..b423a09873b 100644 --- a/spec/helpers/issuables_helper_spec.rb +++ b/spec/helpers/issuables_helper_spec.rb @@ -60,7 +60,7 @@ describe IssuablesHelper do end end - describe 'counter caching based on issuable type and params', :caching do + describe 'counter caching based on issuable type and params', :use_clean_rails_memory_store_caching do let(:params) do { scope: 'created-by-me', diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 487d9800707..c462d9006ea 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -63,7 +63,7 @@ describe ProjectsHelper do end end - describe "#project_list_cache_key", redis: true do + describe "#project_list_cache_key", clean_gitlab_redis_shared_state: true do let(:project) { create(:project) } it "includes the route" do diff --git a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb index fc72df575be..15b3db0ed3d 100644 --- a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb +++ b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Auth::UniqueIpsLimiter, :redis, lib: true do +describe Gitlab::Auth::UniqueIpsLimiter, :clean_gitlab_redis_shared_state, lib: true do include_context 'unique ips sign in limit' let(:user) { create(:user) } diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index 07db6c3a640..0daf41a7c86 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do +describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cache do let!(:project) { create(:project) } let(:pipeline_status) { described_class.new(project) } let(:cache_key) { "projects/#{project.id}/pipeline_status" } @@ -28,8 +28,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class) end - describe 'without a status in redis' do - it 'loads the status from a commit when it was not in redis' do + describe 'without a status in redis_cache' do + it 'loads the status from a commit when it was not in redis_cache' do empty_status = { sha: nil, status: nil, ref: nil } fake_pipeline = described_class.new( project_without_status, @@ -48,9 +48,9 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do described_class.load_in_batch_for_projects([project_without_status]) end - it 'only connects to redis twice' do + it 'only connects to redis_cache twice' do # Once to load, once to store in the cache - expect(Gitlab::Redis).to receive(:with).exactly(2).and_call_original + expect(Gitlab::Redis::Cache).to receive(:with).exactly(2).and_call_original described_class.load_in_batch_for_projects([project_without_status]) @@ -58,9 +58,9 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do end end - describe 'when a status was cached in redis' do + describe 'when a status was cached in redis_cache' do before do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -76,8 +76,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do expect(pipeline_status.ref).to eq(ref) end - it 'only connects to redis once' do - expect(Gitlab::Redis).to receive(:with).exactly(1).and_call_original + it 'only connects to redis_cache once' do + expect(Gitlab::Redis::Cache).to receive(:with).exactly(1).and_call_original described_class.load_in_batch_for_projects([project]) @@ -94,8 +94,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do end describe '.cached_results_for_projects' do - it 'loads a status from redis for all projects' do - Gitlab::Redis.with do |redis| + it 'loads a status from caching for all projects' do + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end @@ -183,7 +183,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do end end - describe "#load_from_project" do + describe "#load_from_project", :clean_gitlab_redis_cache do let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) } it 'reads the status from the pipeline for the commit' do @@ -203,40 +203,40 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do end end - describe "#store_in_cache", :redis do - it "sets the object in redis" do + describe "#store_in_cache", :clean_gitlab_redis_cache do + it "sets the object in caching" do pipeline_status.sha = '123456' pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } + read_sha, read_status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') end end - describe '#store_in_cache_if_needed', :redis do + describe '#store_in_cache_if_needed', :clean_gitlab_redis_cache do it 'stores the state in the cache when the sha is the HEAD of the project' do create(:ci_pipeline, :success, project: project, sha: project.commit.sha) pipeline_status = described_class.load_for_project(project) pipeline_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil expect(ref).not_to be_nil end - it "doesn't store the status in redis when the sha is not the head of the project" do + it "doesn't store the status in redis_cache when the sha is not the head of the project" do other_status = described_class.new( project, pipeline_info: { sha: "123456", status: "failed" } ) other_status.store_in_cache_if_needed - sha, status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } + sha, status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -244,7 +244,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: 'sha', status: 'pending', ref: 'master' }) end @@ -255,7 +255,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -263,20 +263,20 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do end end - describe "with a status in redis", :redis do + describe "with a status in caching", :clean_gitlab_redis_cache do let(:status) { 'success' } let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } let(:ref) { 'master' } before do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end describe '#load_from_cache' do - it 'reads the status from redis' do + it 'reads the status from redis_cache' do pipeline_status.load_from_cache expect(pipeline_status.sha).to eq(sha) @@ -292,10 +292,10 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do end describe '#delete_from_cache' do - it 'deletes values from redis' do + it 'deletes values from redis_cache' do pipeline_status.delete_from_cache - key_exists = Gitlab::Redis.with { |redis| redis.exists(cache_key) } + key_exists = Gitlab::Redis::Cache.with { |redis| redis.exists(cache_key) } expect(key_exists).to be_falsy end diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb index 8813f129ef5..df7d1b5d27a 100644 --- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb +++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb @@ -236,7 +236,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed') old_path, new_path = [nil, nil] - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| rename_info = redis.lpop(key) old_path, new_path = JSON.parse(rename_info) end @@ -268,7 +268,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca key = 'rename:FakeRenameReservedPathMigrationV1:project' stored_renames = nil rename_count = 0 - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| stored_renames = redis.lrange(key, 0, 1) rename_count = redis.llen(key) end diff --git a/spec/lib/gitlab/exclusive_lease_spec.rb b/spec/lib/gitlab/exclusive_lease_spec.rb index 81bbd70ffb8..590d6da4113 100644 --- a/spec/lib/gitlab/exclusive_lease_spec.rb +++ b/spec/lib/gitlab/exclusive_lease_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::ExclusiveLease, type: :redis do +describe Gitlab::ExclusiveLease, type: :clean_gitlab_redis_shared_state do let(:unique_key) { SecureRandom.hex(10) } describe '#try_obtain' do diff --git a/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb new file mode 100644 index 00000000000..3693f52b51b --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::CacheCheck do + include_examples 'simple_check', 'redis_cache_ping', 'RedisCache', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb new file mode 100644 index 00000000000..c69443d205d --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::QueuesCheck do + include_examples 'simple_check', 'redis_queues_ping', 'RedisQueues', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb new file mode 100644 index 00000000000..03afc1cd761 --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::RedisCheck do + include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb new file mode 100644 index 00000000000..b72e152bbe2 --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' +require_relative '../simple_check_shared' + +describe Gitlab::HealthChecks::Redis::SharedStateCheck do + include_examples 'simple_check', 'redis_shared_state_ping', 'RedisSharedState', 'PONG' +end diff --git a/spec/lib/gitlab/health_checks/redis_check_spec.rb b/spec/lib/gitlab/health_checks/redis_check_spec.rb deleted file mode 100644 index 734cdcb893e..00000000000 --- a/spec/lib/gitlab/health_checks/redis_check_spec.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'spec_helper' -require_relative './simple_check_shared' - -describe Gitlab::HealthChecks::RedisCheck do - include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG' -end diff --git a/spec/lib/gitlab/health_checks/simple_check_shared.rb b/spec/lib/gitlab/health_checks/simple_check_shared.rb index 3f871d66034..1abebeac4dd 100644 --- a/spec/lib/gitlab/health_checks/simple_check_shared.rb +++ b/spec/lib/gitlab/health_checks/simple_check_shared.rb @@ -47,7 +47,7 @@ shared_context 'simple_check' do |metrics_prefix, check_name, success_result| allow(described_class).to receive(:check).and_return 'error!' end - it { is_expected.to have_attributes(success: false, message: "unexpected #{check_name} check result: error!") } + it { is_expected.to have_attributes(success: false, message: "unexpected #{described_class.human_name} check result: error!") } end context 'Check is timeouting' do @@ -55,7 +55,7 @@ shared_context 'simple_check' do |metrics_prefix, check_name, success_result| allow(described_class).to receive(:check ).and_return Timeout::Error.new end - it { is_expected.to have_attributes(success: false, message: "#{check_name} check timed out") } + it { is_expected.to have_attributes(success: false, message: "#{described_class.human_name} check timed out") } end end diff --git a/spec/lib/gitlab/performance_bar_spec.rb b/spec/lib/gitlab/performance_bar_spec.rb index 8a586bdbf63..b8a2267f1a4 100644 --- a/spec/lib/gitlab/performance_bar_spec.rb +++ b/spec/lib/gitlab/performance_bar_spec.rb @@ -7,7 +7,7 @@ describe Gitlab::PerformanceBar do described_class.enabled?(user) end - it 'caches the allowed user IDs in cache', :caching do + it 'caches the allowed user IDs in cache', :use_clean_rails_memory_store_caching do expect do expect(described_class.enabled?(user)).to be_truthy end.not_to exceed_query_limit(0) diff --git a/spec/lib/gitlab/redis/cache_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb new file mode 100644 index 00000000000..5a4f17cfcf6 --- /dev/null +++ b/spec/lib/gitlab/redis/cache_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe Gitlab::Redis::Cache do + let(:config_file_name) { "config/redis.cache.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_CACHE_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_cache_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_cache_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.cache.sock" } + let(:new_socket_path) {"/path/to/redis.cache.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_cache_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_cache_new_format_host.yml" } + let(:redis_port) { 6380 } + let(:redis_database) { 10 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_cache_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_CACHE_URL"} + let(:class_redis_url) { Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL } + + include_examples "redis_shared_examples" +end diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb new file mode 100644 index 00000000000..01ca25635a9 --- /dev/null +++ b/spec/lib/gitlab/redis/queues_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe Gitlab::Redis::Queues do + let(:config_file_name) { "config/redis.queues.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_QUEUES_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_queues_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_queues_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.queues.sock" } + let(:new_socket_path) {"/path/to/redis.queues.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_queues_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_queues_new_format_host.yml" } + let(:redis_port) { 6381 } + let(:redis_database) { 11 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_queues_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_QUEUES_URL"} + let(:class_redis_url) { Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL } + + include_examples "redis_shared_examples" +end diff --git a/spec/lib/gitlab/redis/shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb new file mode 100644 index 00000000000..24b73745dc5 --- /dev/null +++ b/spec/lib/gitlab/redis/shared_state_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe Gitlab::Redis::SharedState do + let(:config_file_name) { "config/redis.shared_state.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_SHARED_STATE_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_shared_state_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_shared_state_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.shared_state.sock" } + let(:new_socket_path) {"/path/to/redis.shared_state.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_shared_state_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_shared_state_new_format_host.yml" } + let(:redis_port) { 6382 } + let(:redis_database) { 12 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_shared_state_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_SHARED_STATE_URL"} + let(:class_redis_url) { Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL } + + include_examples "redis_shared_examples" +end diff --git a/spec/lib/gitlab/redis/wrapper_spec.rb b/spec/lib/gitlab/redis/wrapper_spec.rb new file mode 100644 index 00000000000..e1becd0a614 --- /dev/null +++ b/spec/lib/gitlab/redis/wrapper_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe Gitlab::Redis::Wrapper do + let(:config_file_name) { "config/resque.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_CONFIG_FILE" } + let(:config_old_format_socket) { "spec/fixtures/config/redis_old_format_socket.yml" } + let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" } + let(:old_socket_path) {"/path/to/old/redis.sock" } + let(:new_socket_path) {"/path/to/redis.sock" } + let(:config_old_format_host) { "spec/fixtures/config/redis_old_format_host.yml" } + let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" } + let(:redis_port) { 6379 } + let(:redis_database) { 99 } + let(:sentinel_port) { redis_port + 20000 } + let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_config_with_env.yml"} + let(:config_env_variable_url) {"TEST_GITLAB_REDIS_URL"} + let(:class_redis_url) { Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL } + + include_examples "redis_shared_examples" +end diff --git a/spec/lib/gitlab/sidekiq_status_spec.rb b/spec/lib/gitlab/sidekiq_status_spec.rb index 496e50fbae4..c2e77ef6b6c 100644 --- a/spec/lib/gitlab/sidekiq_status_spec.rb +++ b/spec/lib/gitlab/sidekiq_status_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::SidekiqStatus do - describe '.set', :redis do + describe '.set', :clean_gitlab_redis_shared_state do it 'stores the job ID' do described_class.set('123') @@ -14,7 +14,7 @@ describe Gitlab::SidekiqStatus do end end - describe '.unset', :redis do + describe '.unset', :clean_gitlab_redis_shared_state do it 'removes the job ID' do described_class.set('123') described_class.unset('123') @@ -27,7 +27,7 @@ describe Gitlab::SidekiqStatus do end end - describe '.all_completed?', :redis do + describe '.all_completed?', :clean_gitlab_redis_shared_state do it 'returns true if all jobs have been completed' do expect(described_class.all_completed?(%w(123))).to eq(true) end @@ -39,7 +39,7 @@ describe Gitlab::SidekiqStatus do end end - describe '.num_running', :redis do + describe '.num_running', :clean_gitlab_redis_shared_state do it 'returns 0 if all jobs have been completed' do expect(described_class.num_running(%w(123))).to eq(0) end @@ -52,7 +52,7 @@ describe Gitlab::SidekiqStatus do end end - describe '.num_completed', :redis do + describe '.num_completed', :clean_gitlab_redis_shared_state do it 'returns 1 if all jobs have been completed' do expect(described_class.num_completed(%w(123))).to eq(1) end @@ -74,7 +74,7 @@ describe Gitlab::SidekiqStatus do end end - describe 'completed', :redis do + describe 'completed', :clean_gitlab_redis_shared_state do it 'returns the completed job' do expect(described_class.completed_jids(%w(123))).to eq(['123']) end diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb index 187d88c8c58..a4ea0ac59e9 100644 --- a/spec/lib/gitlab/user_activities_spec.rb +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -1,27 +1,27 @@ require 'spec_helper' -describe Gitlab::UserActivities, :redis, lib: true do +describe Gitlab::UserActivities, :clean_gitlab_redis_shared_state, lib: true do let(:now) { Time.now } describe '.record' do context 'with no time given' do - it 'uses Time.now and records an activity in Redis' do + it 'uses Time.now and records an activity in SharedState' do Timecop.freeze do now # eager-load now described_class.record(42) end - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end end context 'with a time given' do - it 'uses the given time and records an activity in Redis' do + it 'uses the given time and records an activity in SharedState' do described_class.record(42, now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end end @@ -31,30 +31,30 @@ describe Gitlab::UserActivities, :redis, lib: true do describe '.delete' do context 'with a single key' do context 'and key exists' do - it 'removes the pair from Redis' do + it 'removes the pair from SharedState' do described_class.record(42, now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(42) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end context 'and key does not exist' do - it 'removes the pair from Redis' do - Gitlab::Redis.with do |redis| + it 'removes the pair from SharedState' do + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end subject.delete(42) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end @@ -63,33 +63,33 @@ describe Gitlab::UserActivities, :redis, lib: true do context 'with multiple keys' do context 'and all keys exist' do - it 'removes the pair from Redis' do + it 'removes the pair from SharedState' do described_class.record(41, now) described_class.record(42, now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end end context 'and some keys does not exist' do - it 'removes the existing pair from Redis' do + it 'removes the existing pair from SharedState' do described_class.record(42, now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) end subject.delete(41, 42) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 493ff3bb5fb..efff0a152a8 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -276,7 +276,7 @@ describe Gitlab::Workhorse, lib: true do end it 'set and notify' do - expect_any_instance_of(Redis).to receive(:publish) + expect_any_instance_of(::Redis).to receive(:publish) .with(described_class::NOTIFICATION_CHANNEL, "test-key=test-value") subject @@ -310,7 +310,7 @@ describe Gitlab::Workhorse, lib: true do end it 'does not notify' do - expect_any_instance_of(Redis).not_to receive(:publish) + expect_any_instance_of(::Redis).not_to receive(:publish) subject end diff --git a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb index 4223d2337a8..5b633dd349b 100644 --- a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb +++ b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb @@ -54,7 +54,7 @@ describe MigrateProcessCommitWorkerJobs do end end - describe '#up', :redis do + describe '#up', :clean_gitlab_redis_shared_state do let(:migration) { described_class.new } def job_count @@ -172,7 +172,7 @@ describe MigrateProcessCommitWorkerJobs do end end - describe '#down', :redis do + describe '#down', :clean_gitlab_redis_shared_state do let(:migration) { described_class.new } def job_count diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb index e3b42b5eac8..063829be546 100644 --- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -3,13 +3,13 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') -describe MigrateUserActivitiesToUsersLastActivityOn, :redis, :truncate do +describe MigrateUserActivitiesToUsersLastActivityOn, :clean_gitlab_redis_shared_state, :truncate do let(:migration) { described_class.new } let!(:user_active_1) { create(:user) } let!(:user_active_2) { create(:user) } def record_activity(user, time) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 76ce558eea0..4b9cce28e0e 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -276,14 +276,14 @@ describe Ci::Runner, models: true do it 'sets a new last_update value when it is called the first time' do last_update = runner.ensure_runner_queue_value - expect_value_in_redis.to eq(last_update) + expect_value_in_queues.to eq(last_update) end it 'does not change if it is not expired and called again' do last_update = runner.ensure_runner_queue_value expect(runner.ensure_runner_queue_value).to eq(last_update) - expect_value_in_redis.to eq(last_update) + expect_value_in_queues.to eq(last_update) end context 'updates runner queue after changing editable value' do @@ -294,7 +294,7 @@ describe Ci::Runner, models: true do end it 'sets a new last_update value' do - expect_value_in_redis.not_to eq(last_update) + expect_value_in_queues.not_to eq(last_update) end end @@ -306,12 +306,12 @@ describe Ci::Runner, models: true do end it 'has an old last_update value' do - expect_value_in_redis.to eq(last_update) + expect_value_in_queues.to eq(last_update) end end - def expect_value_in_redis - Gitlab::Redis.with do |redis| + def expect_value_in_queues + Gitlab::Redis::Queues.with do |redis| runner_queue_key = runner.send(:runner_queue_key) expect(redis.get(runner_queue_key)) end @@ -330,7 +330,7 @@ describe Ci::Runner, models: true do end it 'cleans up the queue' do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| expect(redis.get(queue_key)).to be_nil end end diff --git a/spec/models/concerns/reactive_caching_spec.rb b/spec/models/concerns/reactive_caching_spec.rb index 808247ebfd5..5f9b7e0a367 100644 --- a/spec/models/concerns/reactive_caching_spec.rb +++ b/spec/models/concerns/reactive_caching_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe ReactiveCaching, caching: true do +describe ReactiveCaching, :use_clean_rails_memory_store_caching do include ReactiveCachingHelpers class CacheTest diff --git a/spec/models/project_services/bamboo_service_spec.rb b/spec/models/project_services/bamboo_service_spec.rb index 7b1a554d1fb..99190d763f2 100644 --- a/spec/models/project_services/bamboo_service_spec.rb +++ b/spec/models/project_services/bamboo_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe BambooService, models: true, caching: true do +describe BambooService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers let(:bamboo_url) { 'http://gitlab.com/bamboo' } diff --git a/spec/models/project_services/buildkite_service_spec.rb b/spec/models/project_services/buildkite_service_spec.rb index dd529597067..b4ee6691e67 100644 --- a/spec/models/project_services/buildkite_service_spec.rb +++ b/spec/models/project_services/buildkite_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe BuildkiteService, models: true, caching: true do +describe BuildkiteService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers let(:project) { create(:empty_project) } diff --git a/spec/models/project_services/drone_ci_service_spec.rb b/spec/models/project_services/drone_ci_service_spec.rb index 1400175427f..c9ac256ff38 100644 --- a/spec/models/project_services/drone_ci_service_spec.rb +++ b/spec/models/project_services/drone_ci_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe DroneCiService, models: true, caching: true do +describe DroneCiService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers describe 'associations' do diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index 5ba523a478a..b66bb5321ab 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe KubernetesService, models: true, caching: true do +describe KubernetesService, :use_clean_rails_memory_store_caching, models: true do include KubernetesHelpers include ReactiveCachingHelpers diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb index 37f23b1243c..3fb134ec3b7 100644 --- a/spec/models/project_services/prometheus_service_spec.rb +++ b/spec/models/project_services/prometheus_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe PrometheusService, models: true, caching: true do +describe PrometheusService, :use_clean_rails_memory_store_caching, models: true do include PrometheusHelpers include ReactiveCachingHelpers diff --git a/spec/models/project_services/teamcity_service_spec.rb b/spec/models/project_services/teamcity_service_spec.rb index 6b004098510..3f3a74d0f96 100644 --- a/spec/models/project_services/teamcity_service_spec.rb +++ b/spec/models/project_services/teamcity_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe TeamcityService, models: true, caching: true do +describe TeamcityService, :use_clean_rails_memory_store_caching, models: true do include ReactiveCachingHelpers let(:teamcity_url) { 'http://gitlab.com/teamcity' } diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 99bfab70088..c4bc129dd37 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -899,7 +899,7 @@ describe Project, models: true do end end - describe '.cached_count', caching: true do + describe '.cached_count', :use_clean_rails_memory_store_caching do let(:group) { create(:group, :public) } let!(:project1) { create(:empty_project, :public, group: group) } let!(:project2) { create(:empty_project, :public, group: group) } diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index af305e9b234..7635b0868e7 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -561,7 +561,7 @@ describe Repository, models: true do end end - describe "#changelog", caching: true do + describe "#changelog", :use_clean_rails_memory_store_caching do it 'accepts changelog' do expect(repository.tree).to receive(:blobs).and_return([TestBlob.new('changelog')]) @@ -593,7 +593,7 @@ describe Repository, models: true do end end - describe "#license_blob", caching: true do + describe "#license_blob", :use_clean_rails_memory_store_caching do before do repository.delete_file( user, 'LICENSE', message: 'Remove LICENSE', branch_name: 'master') @@ -638,7 +638,7 @@ describe Repository, models: true do end end - describe '#license_key', caching: true do + describe '#license_key', :use_clean_rails_memory_store_caching do before do repository.delete_file(user, 'LICENSE', message: 'Remove LICENSE', branch_name: 'master') @@ -703,7 +703,7 @@ describe Repository, models: true do end end - describe "#gitlab_ci_yml", caching: true do + describe "#gitlab_ci_yml", :use_clean_rails_memory_store_caching do it 'returns valid file' do files = [TestBlob.new('file'), TestBlob.new('.gitlab-ci.yml'), TestBlob.new('copying')] expect(repository.tree).to receive(:blobs).and_return(files) @@ -1611,7 +1611,7 @@ describe Repository, models: true do end end - describe '#contribution_guide', caching: true do + describe '#contribution_guide', :use_clean_rails_memory_store_caching do it 'returns and caches the output' do expect(repository).to receive(:file_on_head) .with(:contributing) @@ -1625,7 +1625,7 @@ describe Repository, models: true do end end - describe '#gitignore', caching: true do + describe '#gitignore', :use_clean_rails_memory_store_caching do it 'returns and caches the output' do expect(repository).to receive(:file_on_head) .with(:gitignore) @@ -1638,7 +1638,7 @@ describe Repository, models: true do end end - describe '#koding_yml', caching: true do + describe '#koding_yml', :use_clean_rails_memory_store_caching do it 'returns and caches the output' do expect(repository).to receive(:file_on_head) .with(:koding) @@ -1651,7 +1651,7 @@ describe Repository, models: true do end end - describe '#readme', caching: true do + describe '#readme', :use_clean_rails_memory_store_caching do context 'with a non-existing repository' do it 'returns nil' do allow(repository).to receive(:tree).with(:head).and_return(nil) @@ -1822,7 +1822,7 @@ describe Repository, models: true do end end - describe '#cache_method_output', caching: true do + describe '#cache_method_output', :use_clean_rails_memory_store_caching do context 'with a non-existing repository' do let(:value) do repository.cache_method_output(:cats, fallback: 10) do diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 448555d2190..d04162a527f 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -348,7 +348,7 @@ describe User, models: true do end end - describe '#update_tracked_fields!', :redis do + describe '#update_tracked_fields!', :clean_gitlab_redis_shared_state do let(:request) { OpenStruct.new(remote_ip: "127.0.0.1") } let(:user) { create(:user) } @@ -1684,7 +1684,7 @@ describe User, models: true do end end - describe '#refresh_authorized_projects', redis: true do + describe '#refresh_authorized_projects', clean_gitlab_redis_shared_state: true do let(:project1) { create(:empty_project) } let(:project2) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index cde4fa888a0..453eb4683a0 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -168,7 +168,7 @@ describe API::Internal do end end - describe "POST /internal/allowed", :redis do + describe "POST /internal/allowed", :clean_gitlab_redis_shared_state do context "access granted" do before do project.team << [user, :developer] diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb index c34b88f0741..a2368c9d996 100644 --- a/spec/requests/api/users_spec.rb +++ b/spec/requests/api/users_spec.rb @@ -1313,7 +1313,7 @@ describe API::Users do end end - context "user activities", :redis do + context "user activities", :clean_gitlab_redis_shared_state do let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) } let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) } diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 185679e1a0f..d0443a450a2 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -406,7 +406,7 @@ describe 'Git HTTP requests', lib: true do end end - it 'updates the user last activity', :redis do + it 'updates the user last activity', :clean_gitlab_redis_shared_state do expect(user_activity(user)).to be_nil download(path, env) do |response| diff --git a/spec/requests/openid_connect_spec.rb b/spec/requests/openid_connect_spec.rb index 6d1f0b24196..ebba28ba8ce 100644 --- a/spec/requests/openid_connect_spec.rb +++ b/spec/requests/openid_connect_spec.rb @@ -98,7 +98,7 @@ describe 'OpenID Connect requests' do expect(@payload['sub']).to eq hashed_subject end - it 'includes the time of the last authentication', :redis do + it 'includes the time of the last authentication', :clean_gitlab_redis_shared_state do expect(@payload['auth_time']).to eq user.current_sign_in_at.to_i end diff --git a/spec/services/event_create_service_spec.rb b/spec/services/event_create_service_spec.rb index b06cefe071d..8d067c194cc 100644 --- a/spec/services/event_create_service_spec.rb +++ b/spec/services/event_create_service_spec.rb @@ -113,7 +113,7 @@ describe EventCreateService, services: true do end end - describe '#push', :redis do + describe '#push', :clean_gitlab_redis_shared_state do let(:project) { create(:empty_project) } let(:user) { create(:user) } diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index 8e8816870e1..3f77ed10069 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -527,14 +527,18 @@ describe GitPushService, services: true do let(:housekeeping) { Projects::HousekeepingService.new(project) } before do - # Flush any raw Redis data stored by the housekeeping code. - Gitlab::Redis.with { |conn| conn.flushall } + # Flush any raw key-value data stored by the housekeeping code. + Gitlab::Redis::Cache.with { |conn| conn.flushall } + Gitlab::Redis::Queues.with { |conn| conn.flushall } + Gitlab::Redis::SharedState.with { |conn| conn.flushall } allow(Projects::HousekeepingService).to receive(:new).and_return(housekeeping) end after do - Gitlab::Redis.with { |conn| conn.flushall } + Gitlab::Redis::Cache.with { |conn| conn.flushall } + Gitlab::Redis::Queues.with { |conn| conn.flushall } + Gitlab::Redis::SharedState.with { |conn| conn.flushall } end it 'does not perform housekeeping when not needed' do diff --git a/spec/services/milestones/destroy_service_spec.rb b/spec/services/milestones/destroy_service_spec.rb deleted file mode 100644 index 8d1fe3ae2c1..00000000000 --- a/spec/services/milestones/destroy_service_spec.rb +++ /dev/null @@ -1,51 +0,0 @@ -require 'spec_helper' - -describe Milestones::DestroyService, services: true do - let(:user) { create(:user) } - let(:project) { create(:project) } - let(:milestone) { create(:milestone, title: 'Milestone v1.0', project: project) } - let(:issue) { create(:issue, project: project, milestone: milestone) } - let(:merge_request) { create(:merge_request, source_project: project, milestone: milestone) } - - before do - project.team << [user, :master] - end - - def service - described_class.new(project, user, {}) - end - - describe '#execute' do - it 'deletes milestone' do - service.execute(milestone) - - expect { milestone.reload }.to raise_error ActiveRecord::RecordNotFound - end - - it 'deletes milestone id from issuables' do - service.execute(milestone) - - expect(issue.reload.milestone).to be_nil - expect(merge_request.reload.milestone).to be_nil - end - - context 'group milestones' do - let(:group) { create(:group) } - let(:group_milestone) { create(:milestone, group: group) } - - before do - project.update(namespace: group) - group.add_developer(user) - end - - it { expect(service.execute(group_milestone)).to be_nil } - - it 'does not update milestone issuables' do - expect(MergeRequests::UpdateService).not_to receive(:new) - expect(Issues::UpdateService).not_to receive(:new) - - service.execute(group_milestone) - end - end - end -end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 697dc18feb0..b399d3402fd 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -60,14 +60,14 @@ describe Projects::DestroyService, services: true do before do new_user = create(:user) project.team.add_user(new_user, Gitlab::Access::DEVELOPER) - allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Redis::CannotConnectError) + allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(::Redis::CannotConnectError) end it 'keeps project team intact upon an error' do Sidekiq::Testing.inline! do begin destroy_project(project, user, {}) - rescue Redis::CannotConnectError + rescue ::Redis::CannotConnectError end end diff --git a/spec/services/todo_service_spec.rb b/spec/services/todo_service_spec.rb index 175a42a32d9..de41cbab14c 100644 --- a/spec/services/todo_service_spec.rb +++ b/spec/services/todo_service_spec.rb @@ -908,7 +908,7 @@ describe TodoService, services: true do end end - it 'caches the number of todos of a user', :caching do + it 'caches the number of todos of a user', :use_clean_rails_memory_store_caching do create(:todo, :mentioned, user: john_doe, target: issue, project: project) todo = create(:todo, :mentioned, user: john_doe, target: issue, project: project) TodoService.new.mark_todos_as_done([todo], john_doe) diff --git a/spec/services/users/activity_service_spec.rb b/spec/services/users/activity_service_spec.rb index 2e009d4ce1c..e5330d1d3e4 100644 --- a/spec/services/users/activity_service_spec.rb +++ b/spec/services/users/activity_service_spec.rb @@ -7,7 +7,7 @@ describe Users::ActivityService, services: true do subject(:service) { described_class.new(user, 'type') } - describe '#execute', :redis do + describe '#execute', :clean_gitlab_redis_shared_state do context 'when last activity is nil' do before do service.execute diff --git a/spec/services/users/refresh_authorized_projects_service_spec.rb b/spec/services/users/refresh_authorized_projects_service_spec.rb index b65cadbb2f5..1c0f55d2965 100644 --- a/spec/services/users/refresh_authorized_projects_service_spec.rb +++ b/spec/services/users/refresh_authorized_projects_service_spec.rb @@ -8,7 +8,7 @@ describe Users::RefreshAuthorizedProjectsService do let(:user) { project.namespace.owner } let(:service) { described_class.new(user) } - describe '#execute', :redis do + describe '#execute', :clean_gitlab_redis_shared_state do it 'refreshes the authorizations using a lease' do expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) .and_return('foo') diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index a497b8613bb..dc0bdd9f4c7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -91,20 +91,30 @@ RSpec.configure do |config| end end - config.around(:each, :caching) do |example| + config.around(:each, :use_clean_rails_memory_store_caching) do |example| caching_store = Rails.cache - Rails.cache = ActiveSupport::Cache::MemoryStore.new if example.metadata[:caching] + Rails.cache = ActiveSupport::Cache::MemoryStore.new + example.run + Rails.cache = caching_store end - config.around(:each, :redis) do |example| - Gitlab::Redis.with(&:flushall) + config.around(:each, :clean_gitlab_redis_cache) do |example| + Gitlab::Redis::Cache.with(&:flushall) + + example.run + + Gitlab::Redis::Cache.with(&:flushall) + end + + config.around(:each, :clean_gitlab_redis_shared_state) do |example| + Gitlab::Redis::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) example.run - Gitlab::Redis.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) Sidekiq.redis(&:flushall) end diff --git a/spec/lib/gitlab/redis_spec.rb b/spec/support/redis/redis_shared_examples.rb index 593aa5038ad..95a5df181c1 100644 --- a/spec/lib/gitlab/redis_spec.rb +++ b/spec/support/redis/redis_shared_examples.rb @@ -1,12 +1,10 @@ -require 'spec_helper' - -describe Gitlab::Redis do +RSpec.shared_examples "redis_shared_examples" do include StubENV - let(:config) { 'config/resque.yml' } + let(:test_redis_url) { "redis://redishost:#{redis_port}"} before(:each) do - stub_env('GITLAB_REDIS_CONFIG_FILE', Rails.root.join(config).to_s) + stub_env(environment_config_file_name, Rails.root.join(config_file_name)) clear_raw_config end @@ -26,46 +24,40 @@ describe Gitlab::Redis do end context 'when url contains unix socket reference' do - let(:config_old) { 'spec/fixtures/config/redis_old_format_socket.yml' } - let(:config_new) { 'spec/fixtures/config/redis_new_format_socket.yml' } - context 'with old format' do - let(:config) { config_old } + let(:config_file_name) { config_old_format_socket } it 'returns path key instead' do - is_expected.to include(path: '/path/to/old/redis.sock') + is_expected.to include(path: old_socket_path) is_expected.not_to have_key(:url) end end context 'with new format' do - let(:config) { config_new } + let(:config_file_name) { config_new_format_socket } it 'returns path key instead' do - is_expected.to include(path: '/path/to/redis.sock') + is_expected.to include(path: new_socket_path) is_expected.not_to have_key(:url) end end end context 'when url is host based' do - let(:config_old) { 'spec/fixtures/config/redis_old_format_host.yml' } - let(:config_new) { 'spec/fixtures/config/redis_new_format_host.yml' } - context 'with old format' do - let(:config) { config_old } + let(:config_file_name) { config_old_format_host } it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99) + is_expected.to include(host: 'localhost', password: 'mypassword', port: redis_port, db: redis_database) is_expected.not_to have_key(:url) end end context 'with new format' do - let(:config) { config_new } + let(:config_file_name) { config_new_format_host } it 'returns hash with host, port, db, and password' do - is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99) + is_expected.to include(host: 'localhost', password: 'mynewpassword', port: redis_port, db: redis_database) is_expected.not_to have_key(:url) end end @@ -82,21 +74,21 @@ describe Gitlab::Redis do end context 'when yml file with env variable' do - let(:config) { 'spec/fixtures/config/redis_config_with_env.yml' } + let(:config_file_name) { config_with_environment_variable_inside } before do - stub_env('TEST_GITLAB_REDIS_URL', 'redis://redishost:6379') + stub_env(config_env_variable_url, test_redis_url) end it 'reads redis url from env variable' do - expect(described_class.url).to eq 'redis://redishost:6379' + expect(described_class.url).to eq test_redis_url end end end describe '._raw_config' do subject { described_class._raw_config } - let(:config) { '/var/empty/doesnotexist' } + let(:config_file_name) { '/var/empty/doesnotexist' } it 'should be frozen' do expect(subject).to be_frozen @@ -124,7 +116,7 @@ describe Gitlab::Redis do it 'instantiates a connection pool with size 5' do expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original - described_class.with { |_redis| true } + described_class.with { |_redis_shared_example| true } end end @@ -137,7 +129,7 @@ describe Gitlab::Redis do it 'instantiates a connection pool with a size based on the concurrency of the worker' do expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original - described_class.with { |_redis| true } + described_class.with { |_redis_shared_example| true } end end end @@ -146,16 +138,16 @@ describe Gitlab::Redis do subject { described_class.new(Rails.env).sentinels } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + let(:config_file_name) { config_new_format_host } it 'returns an array of hashes with host and port keys' do - is_expected.to include(host: 'localhost', port: 26380) - is_expected.to include(host: 'slave2', port: 26381) + is_expected.to include(host: 'localhost', port: sentinel_port) + is_expected.to include(host: 'slave2', port: sentinel_port) end end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + let(:config_file_name) { config_old_format_host } it 'returns nil' do is_expected.to be_nil @@ -167,7 +159,7 @@ describe Gitlab::Redis do subject { described_class.new(Rails.env).sentinels? } context 'when sentinels are defined' do - let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' } + let(:config_file_name) { config_new_format_host } it 'returns true' do is_expected.to be_truthy @@ -175,7 +167,7 @@ describe Gitlab::Redis do end context 'when sentinels are not defined' do - let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' } + let(:config_file_name) { config_old_format_host } it 'returns false' do is_expected.to be_falsey @@ -187,12 +179,12 @@ describe Gitlab::Redis do it 'returns default redis url when no config file is present' do expect(subject).to receive(:fetch_config) { false } - expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::DEFAULT_REDIS_URL) + expect(subject.send(:raw_config_hash)).to eq(url: class_redis_url ) end it 'returns old-style single url config in a hash' do - expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' } - expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379') + expect(subject).to receive(:fetch_config) { test_redis_url } + expect(subject.send(:raw_config_hash)).to eq(url: test_redis_url) end end diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb index 1986d202c4a..ff0b47899f5 100644 --- a/spec/support/unique_ip_check_shared_examples.rb +++ b/spec/support/unique_ip_check_shared_examples.rb @@ -1,7 +1,9 @@ shared_context 'unique ips sign in limit' do include StubENV before(:each) do - Gitlab::Redis.with(&:flushall) + Gitlab::Redis::Cache.with(&:flushall) + Gitlab::Redis::Queues.with(&:flushall) + Gitlab::Redis::SharedState.with(&:flushall) end before do diff --git a/spec/workers/schedule_update_user_activity_worker_spec.rb b/spec/workers/schedule_update_user_activity_worker_spec.rb index e583c3203aa..32c59381b01 100644 --- a/spec/workers/schedule_update_user_activity_worker_spec.rb +++ b/spec/workers/schedule_update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe ScheduleUpdateUserActivityWorker, :redis do +describe ScheduleUpdateUserActivityWorker, :clean_gitlab_redis_shared_state do let(:now) { Time.now } before do diff --git a/spec/workers/update_user_activity_worker_spec.rb b/spec/workers/update_user_activity_worker_spec.rb index 43e9511f116..268ca1d81f2 100644 --- a/spec/workers/update_user_activity_worker_spec.rb +++ b/spec/workers/update_user_activity_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe UpdateUserActivityWorker, :redis do +describe UpdateUserActivityWorker, :clean_gitlab_redis_shared_state do let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) } let(:user_active_yesterday_1) { create(:user) } let(:user_active_yesterday_2) { create(:user) } @@ -25,7 +25,7 @@ describe UpdateUserActivityWorker, :redis do end end - it 'deletes the pairs from Redis' do + it 'deletes the pairs from SharedState' do data.each { |id, time| Gitlab::UserActivities.record(id, time) } subject.perform(data) |