Merge branch '64251-redis-set-cache-mark-2' into 'master'

Re-introduce the Redis set cache for branch and tag names - but don't enable it yet

See merge request gitlab-org/gitlab-ce!32412

7 files changed, 237 insertions, 3 deletions

diff --git a/app/models/repository.rb b/app/models/repository.rb
index 5cb4b56a114..e5a83366776 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -1134,6 +1134,10 @@ class Repository
     @cache ||= Gitlab::RepositoryCache.new(self)
   end
 
+  def redis_set_cache
+    @redis_set_cache ||= Gitlab::RepositorySetCache.new(self)
+  end
+
   def request_store_cache
     @request_store_cache ||= Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore)
   end
diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb
index e40c366ed02..b2dc92ce010 100644
--- a/lib/gitlab/repository_cache_adapter.rb
+++ b/lib/gitlab/repository_cache_adapter.rb
@@ -23,6 +23,49 @@ module Gitlab
         end
       end
 
+      # Caches and strongly memoizes the method as a Redis Set.
+      #
+      # This only works for methods that do not take any arguments. The method
+      # should return an Array of Strings to be cached.
+      #
+      # In addition to overriding the named method, a "name_include?" method is
+      # defined. This uses the "SISMEMBER" query to efficiently check membership
+      # without needing to load the entire set into memory.
+      #
+      # name     - The name of the method to be cached.
+      # fallback - A value to fall back to if the repository does not exist, or
+      #            in case of a Git error. Defaults to nil.
+      #
+      # It is not safe to use this method prior to the release of 12.3, since
+      # 12.2 does not correctly invalidate the redis set cache value. A mixed
+      # code environment containing both 12.2 and 12.3 nodes breaks, while a
+      # mixed code environment containing both 12.3 and 12.4 nodes will work.
+      def cache_method_as_redis_set(name, fallback: nil)
+        uncached_name = alias_uncached_method(name)
+
+        define_method(name) do
+          cache_method_output_as_redis_set(name, fallback: fallback) do
+            __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+
+        # Attempt to determine whether a value is in the set of values being
+        # cached, by performing a redis SISMEMBERS query if appropriate.
+        #
+        # If the full list is already in-memory, we're better using it directly.
+        #
+        # If the cache is not yet populated, querying it directly will give the
+        # wrong answer. We handle that by querying the full list - which fills
+        # the cache - and using it directly to answer the question.
+        define_method("#{name}_include?") do |value|
+          if strong_memoized?(name) || !redis_set_cache.exist?(name)
+            return __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend
+          end
+
+          redis_set_cache.include?(name, value)
+        end
+      end
+
       # Caches truthy values from the method. All values are strongly memoized,
       # and cached in RequestStore.
       #
@@ -84,6 +127,11 @@ module Gitlab
       raise NotImplementedError
     end
 
+    # RepositorySetCache to be used. Should be overridden by the including class
+    def redis_set_cache
+      raise NotImplementedError
+    end
+
     # List of cached methods. Should be overridden by the including class
     def cached_methods
       raise NotImplementedError
@@ -100,6 +148,18 @@ module Gitlab
       end
     end
 
+    # Caches and strongly memoizes the supplied block as a Redis Set. The result
+    # will be provided as a sorted array.
+    #
+    # name     - The name of the method to be cached.
+    # fallback - A value to fall back to if the repository does not exist, or
+    #            in case of a Git error. Defaults to nil.
+    def cache_method_output_as_redis_set(name, fallback: nil, &block)
+      memoize_method_output(name, fallback: fallback) do
+        redis_set_cache.fetch(name, &block).sort
+      end
+    end
+
     # Caches truthy values from the supplied block. All values are strongly
     # memoized, and cached in RequestStore.
     #
@@ -154,6 +214,7 @@ module Gitlab
         clear_memoization(memoizable_name(name))
       end
 
+      expire_redis_set_method_caches(methods)
       expire_request_store_method_caches(methods)
     end
 
@@ -169,6 +230,10 @@ module Gitlab
       end
     end
 
+    def expire_redis_set_method_caches(methods)
+      methods.each { |name| redis_set_cache.expire(name) }
+    end
+
     # All cached repository methods depend on the existence of a Git repository,
     # so if the repository doesn't exist, we already know not to call it.
     def fallback_early?(method_name)
diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb
new file mode 100644
index 00000000000..6d3ac53a787
--- /dev/null
+++ b/lib/gitlab/repository_set_cache.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# Interface to the Redis-backed cache store for keys that use a Redis set
+module Gitlab
+  class RepositorySetCache
+    attr_reader :repository, :namespace, :expires_in
+
+    def initialize(repository, extra_namespace: nil, expires_in: 2.weeks)
+      @repository = repository
+      @namespace = "#{repository.full_path}:#{repository.project.id}"
+      @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace
+      @expires_in = expires_in
+    end
+
+    def cache_key(type)
+      "#{type}:#{namespace}:set"
+    end
+
+    def expire(key)
+      with { |redis| redis.del(cache_key(key)) }
+    end
+
+    def exist?(key)
+      with { |redis| redis.exists(cache_key(key)) }
+    end
+
+    def read(key)
+      with { |redis| redis.smembers(cache_key(key)) }
+    end
+
+    def write(key, value)
+      full_key = cache_key(key)
+
+      with do |redis|
+        redis.multi do
+          redis.del(full_key)
+
+          # Splitting into groups of 1000 prevents us from creating a too-long
+          # Redis command
+          value.each_slice(1000) { |subset| redis.sadd(full_key, subset) }
+
+          redis.expire(full_key, expires_in)
+        end
+      end
+
+      value
+    end
+
+    def fetch(key, &block)
+      if exist?(key)
+        read(key)
+      else
+        write(key, yield)
+      end
+    end
+
+    def include?(key, value)
+      with { |redis| redis.sismember(cache_key(key), value) }
+    end
+
+    private
+
+    def with(&blk)
+      Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb
index 3021a91dd83..483bfe12c68 100644
--- a/lib/gitlab/utils/strong_memoize.rb
+++ b/lib/gitlab/utils/strong_memoize.rb
@@ -24,13 +24,17 @@ module Gitlab
       #     end
       #
       def strong_memoize(name)
-        if instance_variable_defined?(ivar(name))
+        if strong_memoized?(name)
           instance_variable_get(ivar(name))
         else
           instance_variable_set(ivar(name), yield)
         end
       end
 
+      def strong_memoized?(name)
+        instance_variable_defined?(ivar(name))
+      end
+
       def clear_memoization(name)
         remove_instance_variable(ivar(name)) if instance_variable_defined?(ivar(name))
       end
diff --git a/spec/lib/gitlab/repository_cache_adapter_spec.rb b/spec/lib/gitlab/repository_cache_adapter_spec.rb
index 808eb865a21..fd1338b55a6 100644
--- a/spec/lib/gitlab/repository_cache_adapter_spec.rb
+++ b/spec/lib/gitlab/repository_cache_adapter_spec.rb
@@ -6,6 +6,7 @@ describe Gitlab::RepositoryCacheAdapter do
   let(:project) { create(:project, :repository) }
   let(:repository) { project.repository }
   let(:cache) { repository.send(:cache) }
+  let(:redis_set_cache) { repository.send(:redis_set_cache) }
 
   describe '#cache_method_output', :use_clean_rails_memory_store_caching do
     let(:fallback) { 10 }
@@ -208,9 +209,11 @@ describe Gitlab::RepositoryCacheAdapter do
   describe '#expire_method_caches' do
     it 'expires the caches of the given methods' do
       expect(cache).to receive(:expire).with(:rendered_readme)
-      expect(cache).to receive(:expire).with(:gitignore)
+      expect(cache).to receive(:expire).with(:branch_names)
+      expect(redis_set_cache).to receive(:expire).with(:rendered_readme)
+      expect(redis_set_cache).to receive(:expire).with(:branch_names)
 
-      repository.expire_method_caches(%i(rendered_readme gitignore))
+      repository.expire_method_caches(%i(rendered_readme branch_names))
     end
 
     it 'does not expire caches for non-existent methods' do
diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb
new file mode 100644
index 00000000000..87e51f801e5
--- /dev/null
+++ b/spec/lib/gitlab/repository_set_cache_spec.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do
+  let(:project) { create(:project) }
+  let(:repository) { project.repository }
+  let(:namespace) { "#{repository.full_path}:#{project.id}" }
+  let(:cache) { described_class.new(repository) }
+
+  describe '#cache_key' do
+    subject { cache.cache_key(:foo) }
+
+    it 'includes the namespace' do
+      is_expected.to eq("foo:#{namespace}:set")
+    end
+
+    context 'with a given namespace' do
+      let(:extra_namespace) { 'my:data' }
+      let(:cache) { described_class.new(repository, extra_namespace: extra_namespace) }
+
+      it 'includes the full namespace' do
+        is_expected.to eq("foo:#{namespace}:#{extra_namespace}:set")
+      end
+    end
+  end
+
+  describe '#expire' do
+    it 'expires the given key from the cache' do
+      cache.write(:foo, ['value'])
+
+      expect(cache.read(:foo)).to contain_exactly('value')
+      expect(cache.expire(:foo)).to eq(1)
+      expect(cache.read(:foo)).to be_empty
+    end
+  end
+
+  describe '#exist?' do
+    it 'checks whether the key exists' do
+      expect(cache.exist?(:foo)).to be(false)
+
+      cache.write(:foo, ['value'])
+
+      expect(cache.exist?(:foo)).to be(true)
+    end
+  end
+
+  describe '#fetch' do
+    let(:blk) { -> { ['block value'] } }
+
+    subject { cache.fetch(:foo, &blk) }
+
+    it 'fetches the key from the cache when filled' do
+      cache.write(:foo, ['value'])
+
+      is_expected.to contain_exactly('value')
+    end
+
+    it 'writes the value of the provided block when empty' do
+      cache.expire(:foo)
+
+      is_expected.to contain_exactly('block value')
+      expect(cache.read(:foo)).to contain_exactly('block value')
+    end
+  end
+
+  describe '#include?' do
+    it 'checks inclusion in the Redis set' do
+      cache.write(:foo, ['value'])
+
+      expect(cache.include?(:foo, 'value')).to be(true)
+      expect(cache.include?(:foo, 'bar')).to be(false)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/utils/strong_memoize_spec.rb b/spec/lib/gitlab/utils/strong_memoize_spec.rb
index 26baaf873a8..624e799c5e9 100644
--- a/spec/lib/gitlab/utils/strong_memoize_spec.rb
+++ b/spec/lib/gitlab/utils/strong_memoize_spec.rb
@@ -52,6 +52,22 @@ describe Gitlab::Utils::StrongMemoize do
     end
   end
 
+  describe '#strong_memoized?' do
+    let(:value) { :anything }
+
+    subject { object.strong_memoized?(:method_name) }
+
+    it 'returns false if the value is uncached' do
+      is_expected.to be(false)
+    end
+
+    it 'returns true if the value is cached' do
+      object.method_name
+
+      is_expected.to be(true)
+    end
+  end
+
   describe '#clear_memoization' do
     let(:value) { 'mepmep' }