Add a memory cache local to the thread to reduce Redis load

Loading `ApplicationSetting` from Redis was responsible for at least 50%
of the CPU load of the Redis cluster on GitLab.com. Since these values
generally don't change very much, we can load this from the database and
cache it in memory, skipping Redis altogther. We use
`ActiveSupport::Cache::MemoryStore` as a drop-in replacement for
`RedisCacheStore` even though we probably don't need synchronized access
within `Thread.current`.

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/63977

6 files changed, 40 insertions, 3 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index fbd8036653a..8e558487c1c 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -272,4 +272,12 @@ class ApplicationSetting < ApplicationRecord
     # We already have an ApplicationSetting record, so just return it.
     current_without_cache
   end
+
+  # By default, the backend is Rails.cache, which uses
+  # ActiveSupport::Cache::RedisStore. Since loading ApplicationSetting
+  # can cause a significant amount of load on Redis, let's cache it in
+  # memory.
+  def self.cache_backend
+    Gitlab::ThreadMemoryCache.cache_backend
+  end
 end
diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb
index 3d60f6924c1..8cbf4bcfaf7 100644
--- a/app/models/concerns/cacheable_attributes.rb
+++ b/app/models/concerns/cacheable_attributes.rb
@@ -36,7 +36,7 @@ module CacheableAttributes
     end
 
     def retrieve_from_cache
-      record = Rails.cache.read(cache_key)
+      record = cache_backend.read(cache_key)
       ensure_cache_setup if record.present?
 
       record
@@ -58,7 +58,7 @@ module CacheableAttributes
     end
 
     def expire
-      Rails.cache.delete(cache_key)
+      cache_backend.delete(cache_key)
     rescue
       # Gracefully handle when Redis is not available. For example,
       # omnibus may fail here during gitlab:assets:compile.
@@ -69,9 +69,13 @@ module CacheableAttributes
       # to be loaded when read from cache: https://github.com/rails/rails/issues/27348
       define_attribute_methods
     end
+
+    def cache_backend
+      Rails.cache
+    end
   end
 
   def cache!
-    Rails.cache.write(self.class.cache_key, self, expires_in: 1.minute)
+    self.class.cache_backend.write(self.class.cache_key, self, expires_in: 1.minute)
   end
 end
diff --git a/changelogs/unreleased/sh-add-thread-memory-cache.yml b/changelogs/unreleased/sh-add-thread-memory-cache.yml
new file mode 100644
index 00000000000..025ad6d9f14
--- /dev/null
+++ b/changelogs/unreleased/sh-add-thread-memory-cache.yml
@@ -0,0 +1,5 @@
+---
+title: Add a memory cache local to the thread to reduce Redis load
+merge_request: 30233
+author:
+type: performance
diff --git a/config/initializers/0_thread_cache.rb b/config/initializers/0_thread_cache.rb
new file mode 100644
index 00000000000..feb8057132e
--- /dev/null
+++ b/config/initializers/0_thread_cache.rb
@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Gitlab::ThreadMemoryCache.cache_backend
diff --git a/lib/gitlab/thread_memory_cache.rb b/lib/gitlab/thread_memory_cache.rb
new file mode 100644
index 00000000000..7f363dc7feb
--- /dev/null
+++ b/lib/gitlab/thread_memory_cache.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class ThreadMemoryCache
+    THREAD_KEY = :thread_memory_cache
+
+    def self.cache_backend
+      # Note ActiveSupport::Cache::MemoryStore is thread-safe.  Since
+      # each backend is local per thread we probably don't need to worry
+      # about synchronizing access, but this is a drop-in replacement
+      # for ActiveSupport::Cache::RedisStore.
+      Thread.current[THREAD_KEY] ||= ActiveSupport::Cache::MemoryStore.new
+    end
+  end
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 3bd2408dc72..62fdc039b5e 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -139,6 +139,8 @@ RSpec.configure do |config|
     allow(Feature).to receive(:enabled?)
       .with(:force_autodevops_on_by_default, anything)
       .and_return(false)
+
+    Gitlab::ThreadMemoryCache.cache_backend.clear
   end
 
   config.around(:example, :quarantine) do