Opt out of L1 cache for operations that perform invalidationan-multi-level-cache-store

12 files changed, 128 insertions, 62 deletions

diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb
index d91be73d6f0..068f7b02320 100644
--- a/app/models/concerns/reactive_caching.rb
+++ b/app/models/concerns/reactive_caching.rb
@@ -114,7 +114,7 @@ module ReactiveCaching
       keep_alive_reactive_cache!(*args)
 
       begin
-        data = Rails.cache.read(full_reactive_cache_key(*args))
+        data = Gitlab::Cache::Store.main.read(full_reactive_cache_key(*args))
         yield data unless data.nil?
       rescue InvalidateReactiveCache
         refresh_reactive_cache!(*args)
@@ -123,8 +123,8 @@ module ReactiveCaching
     end
 
     def clear_reactive_cache!(*args)
-      Rails.cache.delete(full_reactive_cache_key(*args))
-      Rails.cache.delete(alive_reactive_cache_key(*args))
+      Gitlab::Cache::Store.main.delete(full_reactive_cache_key(*args))
+      Gitlab::Cache::Store.main.delete(alive_reactive_cache_key(*args))
     end
 
     def exclusively_update_reactive_cache!(*args)
@@ -133,8 +133,8 @@ module ReactiveCaching
           enqueuing_update(*args) do
             key = full_reactive_cache_key(*args)
             new_value = calculate_reactive_cache(*args)
-            old_value = Rails.cache.read(key)
-            Rails.cache.write(key, new_value)
+            old_value = Gitlab::Cache::Store.main.read(key)
+            Gitlab::Cache::Store.main.write(key, new_value)
             reactive_cache_updated(*args) if new_value != old_value
           end
         end
@@ -150,7 +150,7 @@ module ReactiveCaching
     end
 
     def keep_alive_reactive_cache!(*args)
-      Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime)
+      Gitlab::Cache::Store.main.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime)
     end
 
     def full_reactive_cache_key(*qualifiers)
@@ -174,9 +174,9 @@ module ReactiveCaching
 
     def within_reactive_cache_lifetime?(*args)
       if Feature.enabled?(:reactive_caching_check_key_exists, default_enabled: true)
-        Rails.cache.exist?(alive_reactive_cache_key(*args))
+        Gitlab::Cache::Store.main.exist?(alive_reactive_cache_key(*args))
       else
-        !!Rails.cache.read(alive_reactive_cache_key(*args))
+        !!Gitlab::Cache::Store.main.read(alive_reactive_cache_key(*args))
       end
     end
 
diff --git a/app/models/user.rb b/app/models/user.rb
index 0fd3daa3383..e55abdb8835 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -1282,31 +1282,31 @@ class User < ApplicationRecord
   end
 
   def assigned_open_merge_requests_count(force: false)
-    Rails.cache.fetch(['users', id, 'assigned_open_merge_requests_count'], force: force, expires_in: 20.minutes) do
+    Gitlab::Cache::Store.main.fetch(['users', id, 'assigned_open_merge_requests_count'], force: force, expires_in: 20.minutes) do
       MergeRequestsFinder.new(self, assignee_id: self.id, state: 'opened', non_archived: true).execute.count
     end
   end
 
   def assigned_open_issues_count(force: false)
-    Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force, expires_in: 20.minutes) do
+    Gitlab::Cache::Store.main.fetch(['users', id, 'assigned_open_issues_count'], force: force, expires_in: 20.minutes) do
       IssuesFinder.new(self, assignee_id: self.id, state: 'opened', non_archived: true).execute.count
     end
   end
 
   def todos_done_count(force: false)
-    Rails.cache.fetch(['users', id, 'todos_done_count'], force: force, expires_in: 20.minutes) do
+    Gitlab::Cache::Store.main.fetch(['users', id, 'todos_done_count'], force: force, expires_in: 20.minutes) do
       TodosFinder.new(self, state: :done).execute.count
     end
   end
 
   def todos_pending_count(force: false)
-    Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force, expires_in: 20.minutes) do
+    Gitlab::Cache::Store.main.fetch(['users', id, 'todos_pending_count'], force: force, expires_in: 20.minutes) do
       TodosFinder.new(self, state: :pending).execute.count
     end
   end
 
   def personal_projects_count(force: false)
-    Rails.cache.fetch(['users', id, 'personal_projects_count'], force: force, expires_in: 24.hours, raw: true) do
+    Gitlab::Cache::Store.main.fetch(['users', id, 'personal_projects_count'], force: force, expires_in: 24.hours, raw: true) do
       personal_projects.count
     end.to_i
   end
@@ -1325,23 +1325,23 @@ class User < ApplicationRecord
   end
 
   def invalidate_issue_cache_counts
-    Rails.cache.delete(['users', id, 'assigned_open_issues_count'])
+    Gitlab::Cache::Store.main.delete(['users', id, 'assigned_open_issues_count'])
   end
 
   def invalidate_merge_request_cache_counts
-    Rails.cache.delete(['users', id, 'assigned_open_merge_requests_count'])
+    Gitlab::Cache::Store.main.delete(['users', id, 'assigned_open_merge_requests_count'])
   end
 
   def invalidate_todos_done_count
-    Rails.cache.delete(['users', id, 'todos_done_count'])
+    Gitlab::Cache::Store.main.delete(['users', id, 'todos_done_count'])
   end
 
   def invalidate_todos_pending_count
-    Rails.cache.delete(['users', id, 'todos_pending_count'])
+    Gitlab::Cache::Store.main.delete(['users', id, 'todos_pending_count'])
   end
 
   def invalidate_personal_projects_count
-    Rails.cache.delete(['users', id, 'personal_projects_count'])
+    Gitlab::Cache::Store.main.delete(['users', id, 'personal_projects_count'])
   end
 
   # This is copied from Devise::Models::Lockable#valid_for_authentication?, as our auth
diff --git a/app/services/base_count_service.rb b/app/services/base_count_service.rb
index ad1647842b8..8de3604e8d2 100644
--- a/app/services/base_count_service.rb
+++ b/app/services/base_count_service.rb
@@ -11,11 +11,11 @@ class BaseCountService
   end
 
   def count
-    Rails.cache.fetch(cache_key, cache_options) { uncached_count }.to_i
+    Gitlab::Cache::Store.main.fetch(cache_key, cache_options) { uncached_count }.to_i
   end
 
   def count_stored?
-    Rails.cache.read(cache_key).present?
+    Gitlab::Cache::Store.main.read(cache_key).present?
   end
 
   def refresh_cache(&block)
@@ -27,7 +27,7 @@ class BaseCountService
   end
 
   def delete_cache
-    Rails.cache.delete(cache_key)
+    Gitlab::Cache::Store.main.delete(cache_key)
   end
 
   def raw?
@@ -45,6 +45,6 @@ class BaseCountService
   end
 
   def update_cache_for_key(key, &block)
-    Rails.cache.write(key, block_given? ? yield : uncached_count, raw: raw?)
+    Gitlab::Cache::Store.main.write(key, block_given? ? yield : uncached_count, raw: raw?)
   end
 end
diff --git a/app/services/users/last_push_event_service.rb b/app/services/users/last_push_event_service.rb
index b3980b8e32c..f06c264b1ee 100644
--- a/app/services/users/last_push_event_service.rb
+++ b/app/services/users/last_push_event_service.rb
@@ -9,7 +9,7 @@ module Users
       @user = user
     end
 
-    # Caches the given push event for the current user in the Rails cache.
+    # Caches the given push event for the current user in the Gitlab::Cache::Store.main.
     #
     # event - An instance of PushEvent to cache.
     def cache_last_push_event(event)
@@ -50,7 +50,7 @@ module Users
         # We don't want to keep querying the same data over and over when a
         # merge request has been created, thus we remove the key if no event
         # (meaning an MR was created) is returned.
-        Rails.cache.delete(cache_key)
+        Gitlab::Cache::Store.main.delete(cache_key)
       end
 
       event
@@ -75,13 +75,13 @@ module Users
     end
 
     def get_key(key)
-      Rails.cache.read(key, raw: true)
+      Gitlab::Cache::Store.main.read(key, raw: true)
     end
 
     def set_key(key, value)
       # We're using raw values here since this takes up less space and we don't
       # store complex objects.
-      Rails.cache.write(key, value, raw: true, expires_in: EXPIRATION)
+      Gitlab::Cache::Store.main.write(key, value, raw: true, expires_in: EXPIRATION)
     end
   end
 end
diff --git a/config/application.rb b/config/application.rb
index d6a2a1bb475..3da204ac02e 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -15,6 +15,7 @@ module Gitlab
     require_dependency Rails.root.join('lib/gitlab')
     require_dependency Rails.root.join('lib/gitlab/redis/wrapper')
     require_dependency Rails.root.join('lib/gitlab/redis/cache')
+    require_dependency Rails.root.join('lib/gitlab/cache/store')
     require_dependency Rails.root.join('lib/gitlab/redis/queues')
     require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
     require_dependency Rails.root.join('lib/gitlab/request_context')
@@ -228,25 +229,7 @@ module Gitlab
       end
     end
 
-    # Use caching across all environments
-    caching_config_hash = Gitlab::Redis::Cache.params
-    caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
-    caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
-    if Sidekiq.server? # threaded context
-      caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
-      caching_config_hash[:pool_timeout] = 1
-    end
-
-    l1_cache_size_mb = ENV['GITLAB_L1_RAILS_CACHE_SIZE_MB'].to_i
-    if l1_cache_size_mb > 0 # rubocop:disable Style/ConditionalAssignment
-      config.cache_store = :level2, {
-        L1: [:memory_store, size: l1_cache_size_mb.megabytes, expires_in: 1.minute, race_condition_ttl: 1.second],
-        L2: [:redis_store, caching_config_hash]
-      }
-    else
-      config.cache_store = :redis_store, caching_config_hash
-    end
-
+    config.cache_store = Gitlab::Cache::Store.cache
     config.active_job.queue_adapter = :sidekiq
 
     # This is needed for gitlab-shell
diff --git a/lib/gitlab/auth/o_auth/session.rb b/lib/gitlab/auth/o_auth/session.rb
index 4925b107042..9e2a66286b3 100644
--- a/lib/gitlab/auth/o_auth/session.rb
+++ b/lib/gitlab/auth/o_auth/session.rb
@@ -6,15 +6,15 @@ module Gitlab
     module OAuth
       module Session
         def self.create(provider, ticket)
-          Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration)
+          Gitlab::Cache::Store.main.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration)
         end
 
         def self.destroy(provider, ticket)
-          Rails.cache.delete("gitlab:#{provider}:#{ticket}")
+          Gitlab::Cache::Store.main.delete("gitlab:#{provider}:#{ticket}")
         end
 
         def self.valid?(provider, ticket)
-          Rails.cache.read("gitlab:#{provider}:#{ticket}").present?
+          Gitlab::Cache::Store.main.read("gitlab:#{provider}:#{ticket}").present?
         end
       end
     end
diff --git a/lib/gitlab/cache/store.rb b/lib/gitlab/cache/store.rb
new file mode 100644
index 00000000000..c5bdb4143ed
--- /dev/null
+++ b/lib/gitlab/cache/store.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cache
+    # See https://docs.gitlab.com/ee/development/utilities.html#cachestore
+    module Store
+      def self.cache
+        @level1 || init_level_1_cache
+      end
+
+      # Allow the l1 cache to be overridden for testing purposes
+      def self.cache=(rhs)
+        @level1 = rhs
+      end
+
+      def self.main
+        @level2 || init_level_2_cache
+      end
+
+      # Allow the main cache to be overridden for testing purposes
+      def self.main=(rhs)
+        @level2 = rhs
+      end
+
+      def self.init_level_1_cache
+        init_cache
+
+        @level1
+      end
+      private_class_method :init_level_1_cache
+
+      def self.init_level_2_cache
+        init_cache
+
+        @level2
+      end
+      private_class_method :init_level_2_cache
+
+      def self.init_cache
+        l1_cache_size_mb = ENV['GITLAB_L1_RAILS_CACHE_SIZE_MB'].to_i
+
+        if l1_cache_size_mb > 0
+          @level2 = create_redis_cache_store
+          @level1 = ActiveSupport::Cache.lookup_store(:level2, {
+            L1: create_inmemory_store(l1_cache_size_mb),
+            L2: @level2
+          })
+        else
+          @level1 = @level2 = create_redis_cache_store
+        end
+      end
+      private_class_method :init_cache
+
+      def self.create_inmemory_store(l1_cache_size_mb)
+        ActiveSupport::Cache.lookup_store(:memory_store, size: l1_cache_size_mb.megabytes, expires_in: 1.minute, race_condition_ttl: 1.second)
+      end
+      private_class_method :create_inmemory_store
+
+      def self.create_redis_cache_store
+        # Use caching across all environments
+        caching_config_hash = Gitlab::Redis::Cache.params
+        caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
+        caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
+        if Sidekiq.server? # threaded context
+          caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
+          caching_config_hash[:pool_timeout] = 1
+        end
+
+        ActiveSupport::Cache.lookup_store(:redis_store, caching_config_hash)
+      end
+      private_class_method :create_redis_cache_store
+    end
+
+
+
+  end
+end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index e4390771db2..b16f00df36e 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -6,7 +6,7 @@ module Gitlab
       delegate :diffable, to: :@diff_collection
       delegate :diff_options, to: :@diff_collection
 
-      def initialize(diff_collection, backend: Rails.cache)
+      def initialize(diff_collection, backend: Gitlab::Cache::Store.main)
         @backend = backend
         @diff_collection = diff_collection
       end
diff --git a/lib/gitlab/metrics/dashboard/cache.rb b/lib/gitlab/metrics/dashboard/cache.rb
index a9ccf0fea9b..662737496cb 100644
--- a/lib/gitlab/metrics/dashboard/cache.rb
+++ b/lib/gitlab/metrics/dashboard/cache.rb
@@ -14,16 +14,16 @@ module Gitlab
           def fetch(key)
             register_key(key)
 
-            Rails.cache.fetch(key) { yield }
+            Gitlab::Cache::Store.main.fetch(key) { yield }
           end
 
           # Resets all dashboard caches, such that all
           # dashboard content will be loaded from source on
           # subsequent dashboard calls.
           def delete_all!
-            all_keys.each { |key| Rails.cache.delete(key) }
+            all_keys.each { |key| Gitlab::Cache::Store.main.delete(key) }
 
-            Rails.cache.delete(CACHE_KEYS)
+            Gitlab::Cache::Store.main.delete(CACHE_KEYS)
           end
 
           private
@@ -31,11 +31,11 @@ module Gitlab
           def register_key(key)
             new_keys = all_keys.add(key).to_a.join('|')
 
-            Rails.cache.write(CACHE_KEYS, new_keys)
+            Gitlab::Cache::Store.main.write(CACHE_KEYS, new_keys)
           end
 
           def all_keys
-            Set.new(Rails.cache.read(CACHE_KEYS)&.split('|'))
+            Set.new(Gitlab::Cache::Store.main.read(CACHE_KEYS)&.split('|'))
           end
         end
       end
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index 5cfa64fd764..1e741032e0e 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -2785,7 +2785,7 @@ describe User do
 
       expect(cache_mock).to receive(:delete).with(['users', user.id, 'assigned_open_issues_count'])
 
-      allow(Rails).to receive(:cache).and_return(cache_mock)
+      allow(Gitlab::Cache::Store).to receive(:main).and_return(cache_mock)
 
       user.invalidate_issue_cache_counts
     end
@@ -2799,7 +2799,7 @@ describe User do
 
       expect(cache_mock).to receive(:delete).with(['users', user.id, 'assigned_open_merge_requests_count'])
 
-      allow(Rails).to receive(:cache).and_return(cache_mock)
+      allow(Gitlab::Cache::Store).to receive(:main).and_return(cache_mock)
 
       user.invalidate_merge_request_cache_counts
     end
@@ -2813,7 +2813,7 @@ describe User do
 
       expect(cache_mock).to receive(:delete).with(['users', user.id, 'personal_projects_count'])
 
-      allow(Rails).to receive(:cache).and_return(cache_mock)
+      allow(Gitlab::Cache::Store).to receive(:main).and_return(cache_mock)
 
       user.invalidate_personal_projects_count
     end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index a6fb172e79b..4f2021cdad4 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -182,11 +182,17 @@ RSpec.configure do |config|
 
   config.around(:each, :use_clean_rails_memory_store_caching) do |example|
     caching_store = Rails.cache
+    level1_store = Gitlab::Cache::Store.cache
+    level2_store = Gitlab::Cache::Store.main
     Rails.cache = ActiveSupport::Cache::MemoryStore.new
+    Gitlab::Cache::Store.cache = Rails.cache
+    Gitlab::Cache::Store.main = Rails.cache
 
     example.run
 
     Rails.cache = caching_store
+    Gitlab::Cache::Store.cache = level1_store
+    Gitlab::Cache::Store.main = level2_store
   end
 
   config.around(:each, :clean_gitlab_redis_cache) do |example|
diff --git a/spec/support/helpers/reactive_caching_helpers.rb b/spec/support/helpers/reactive_caching_helpers.rb
index 528da37e8cf..743d16d91d7 100644
--- a/spec/support/helpers/reactive_caching_helpers.rb
+++ b/spec/support/helpers/reactive_caching_helpers.rb
@@ -20,24 +20,24 @@ module ReactiveCachingHelpers
   end
 
   def read_reactive_cache(subject, *qualifiers)
-    Rails.cache.read(reactive_cache_key(subject, *qualifiers))
+    Gitlab::Cache::Store.main.read(reactive_cache_key(subject, *qualifiers))
   end
 
   def write_reactive_cache(subject, data, *qualifiers)
     start_reactive_cache_lifetime(subject, *qualifiers)
-    Rails.cache.write(reactive_cache_key(subject, *qualifiers), data)
+    Gitlab::Cache::Store.main.write(reactive_cache_key(subject, *qualifiers), data)
   end
 
   def reactive_cache_alive?(subject, *qualifiers)
-    Rails.cache.read(alive_reactive_cache_key(subject, *qualifiers))
+    Gitlab::Cache::Store.main.read(alive_reactive_cache_key(subject, *qualifiers))
   end
 
   def invalidate_reactive_cache(subject, *qualifiers)
-    Rails.cache.delete(alive_reactive_cache_key(subject, *qualifiers))
+    Gitlab::Cache::Store.main.delete(alive_reactive_cache_key(subject, *qualifiers))
   end
 
   def start_reactive_cache_lifetime(subject, *qualifiers)
-    Rails.cache.write(alive_reactive_cache_key(subject, *qualifiers), true)
+    Gitlab::Cache::Store.main.write(alive_reactive_cache_key(subject, *qualifiers), true)
   end
 
   def expect_reactive_cache_update_queued(subject)