diff options
author | Stan Hu <stanhu@gmail.com> | 2019-09-10 17:38:42 +0000 |
---|---|---|
committer | Stan Hu <stanhu@gmail.com> | 2019-09-10 17:38:42 +0000 |
commit | 4bcde77f66d94696b204b141ad5a38bf127a5215 (patch) | |
tree | 1d98276609bbc63c5c309c4ae2c52bdbe95bc487 | |
parent | 0498ec89a1b44caad91603215d1ea96aaa4e1eb8 (diff) | |
parent | 034f0340a3e5c4c593916526246fccf0737ab0e3 (diff) | |
download | gitlab-ce-4bcde77f66d94696b204b141ad5a38bf127a5215.tar.gz |
Merge branch '64251-redis-set-cache-mark-2' into 'master'
Re-introduce the Redis set cache for branch and tag names - but don't enable it yet
See merge request gitlab-org/gitlab-ce!32412
-rw-r--r-- | app/models/repository.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/repository_cache_adapter.rb | 65 | ||||
-rw-r--r-- | lib/gitlab/repository_set_cache.rb | 67 | ||||
-rw-r--r-- | lib/gitlab/utils/strong_memoize.rb | 6 | ||||
-rw-r--r-- | spec/lib/gitlab/repository_cache_adapter_spec.rb | 7 | ||||
-rw-r--r-- | spec/lib/gitlab/repository_set_cache_spec.rb | 75 | ||||
-rw-r--r-- | spec/lib/gitlab/utils/strong_memoize_spec.rb | 16 |
7 files changed, 237 insertions, 3 deletions
diff --git a/app/models/repository.rb b/app/models/repository.rb index 5cb4b56a114..e5a83366776 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1134,6 +1134,10 @@ class Repository @cache ||= Gitlab::RepositoryCache.new(self) end + def redis_set_cache + @redis_set_cache ||= Gitlab::RepositorySetCache.new(self) + end + def request_store_cache @request_store_cache ||= Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore) end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index e40c366ed02..b2dc92ce010 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -23,6 +23,49 @@ module Gitlab end end + # Caches and strongly memoizes the method as a Redis Set. + # + # This only works for methods that do not take any arguments. The method + # should return an Array of Strings to be cached. + # + # In addition to overriding the named method, a "name_include?" method is + # defined. This uses the "SISMEMBER" query to efficiently check membership + # without needing to load the entire set into memory. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + # + # It is not safe to use this method prior to the release of 12.3, since + # 12.2 does not correctly invalidate the redis set cache value. A mixed + # code environment containing both 12.2 and 12.3 nodes breaks, while a + # mixed code environment containing both 12.3 and 12.4 nodes will work. + def cache_method_as_redis_set(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output_as_redis_set(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + + # Attempt to determine whether a value is in the set of values being + # cached, by performing a redis SISMEMBERS query if appropriate. + # + # If the full list is already in-memory, we're better using it directly. + # + # If the cache is not yet populated, querying it directly will give the + # wrong answer. We handle that by querying the full list - which fills + # the cache - and using it directly to answer the question. + define_method("#{name}_include?") do |value| + if strong_memoized?(name) || !redis_set_cache.exist?(name) + return __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend + end + + redis_set_cache.include?(name, value) + end + end + # Caches truthy values from the method. All values are strongly memoized, # and cached in RequestStore. # @@ -84,6 +127,11 @@ module Gitlab raise NotImplementedError end + # RepositorySetCache to be used. Should be overridden by the including class + def redis_set_cache + raise NotImplementedError + end + # List of cached methods. Should be overridden by the including class def cached_methods raise NotImplementedError @@ -100,6 +148,18 @@ module Gitlab end end + # Caches and strongly memoizes the supplied block as a Redis Set. The result + # will be provided as a sorted array. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output_as_redis_set(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + redis_set_cache.fetch(name, &block).sort + end + end + # Caches truthy values from the supplied block. All values are strongly # memoized, and cached in RequestStore. # @@ -154,6 +214,7 @@ module Gitlab clear_memoization(memoizable_name(name)) end + expire_redis_set_method_caches(methods) expire_request_store_method_caches(methods) end @@ -169,6 +230,10 @@ module Gitlab end end + def expire_redis_set_method_caches(methods) + methods.each { |name| redis_set_cache.expire(name) } + end + # All cached repository methods depend on the existence of a Git repository, # so if the repository doesn't exist, we already know not to call it. def fallback_early?(method_name) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb new file mode 100644 index 00000000000..6d3ac53a787 --- /dev/null +++ b/lib/gitlab/repository_set_cache.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Interface to the Redis-backed cache store for keys that use a Redis set +module Gitlab + class RepositorySetCache + attr_reader :repository, :namespace, :expires_in + + def initialize(repository, extra_namespace: nil, expires_in: 2.weeks) + @repository = repository + @namespace = "#{repository.full_path}:#{repository.project.id}" + @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace + @expires_in = expires_in + end + + def cache_key(type) + "#{type}:#{namespace}:set" + end + + def expire(key) + with { |redis| redis.del(cache_key(key)) } + end + + def exist?(key) + with { |redis| redis.exists(cache_key(key)) } + end + + def read(key) + with { |redis| redis.smembers(cache_key(key)) } + end + + def write(key, value) + full_key = cache_key(key) + + with do |redis| + redis.multi do + redis.del(full_key) + + # Splitting into groups of 1000 prevents us from creating a too-long + # Redis command + value.each_slice(1000) { |subset| redis.sadd(full_key, subset) } + + redis.expire(full_key, expires_in) + end + end + + value + end + + def fetch(key, &block) + if exist?(key) + read(key) + else + write(key, yield) + end + end + + def include?(key, value) + with { |redis| redis.sismember(cache_key(key), value) } + end + + private + + def with(&blk) + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 3021a91dd83..483bfe12c68 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -24,13 +24,17 @@ module Gitlab # end # def strong_memoize(name) - if instance_variable_defined?(ivar(name)) + if strong_memoized?(name) instance_variable_get(ivar(name)) else instance_variable_set(ivar(name), yield) end end + def strong_memoized?(name) + instance_variable_defined?(ivar(name)) + end + def clear_memoization(name) remove_instance_variable(ivar(name)) if instance_variable_defined?(ivar(name)) end diff --git a/spec/lib/gitlab/repository_cache_adapter_spec.rb b/spec/lib/gitlab/repository_cache_adapter_spec.rb index 808eb865a21..fd1338b55a6 100644 --- a/spec/lib/gitlab/repository_cache_adapter_spec.rb +++ b/spec/lib/gitlab/repository_cache_adapter_spec.rb @@ -6,6 +6,7 @@ describe Gitlab::RepositoryCacheAdapter do let(:project) { create(:project, :repository) } let(:repository) { project.repository } let(:cache) { repository.send(:cache) } + let(:redis_set_cache) { repository.send(:redis_set_cache) } describe '#cache_method_output', :use_clean_rails_memory_store_caching do let(:fallback) { 10 } @@ -208,9 +209,11 @@ describe Gitlab::RepositoryCacheAdapter do describe '#expire_method_caches' do it 'expires the caches of the given methods' do expect(cache).to receive(:expire).with(:rendered_readme) - expect(cache).to receive(:expire).with(:gitignore) + expect(cache).to receive(:expire).with(:branch_names) + expect(redis_set_cache).to receive(:expire).with(:rendered_readme) + expect(redis_set_cache).to receive(:expire).with(:branch_names) - repository.expire_method_caches(%i(rendered_readme gitignore)) + repository.expire_method_caches(%i(rendered_readme branch_names)) end it 'does not expire caches for non-existent methods' do diff --git a/spec/lib/gitlab/repository_set_cache_spec.rb b/spec/lib/gitlab/repository_set_cache_spec.rb new file mode 100644 index 00000000000..87e51f801e5 --- /dev/null +++ b/spec/lib/gitlab/repository_set_cache_spec.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::RepositorySetCache, :clean_gitlab_redis_cache do + let(:project) { create(:project) } + let(:repository) { project.repository } + let(:namespace) { "#{repository.full_path}:#{project.id}" } + let(:cache) { described_class.new(repository) } + + describe '#cache_key' do + subject { cache.cache_key(:foo) } + + it 'includes the namespace' do + is_expected.to eq("foo:#{namespace}:set") + end + + context 'with a given namespace' do + let(:extra_namespace) { 'my:data' } + let(:cache) { described_class.new(repository, extra_namespace: extra_namespace) } + + it 'includes the full namespace' do + is_expected.to eq("foo:#{namespace}:#{extra_namespace}:set") + end + end + end + + describe '#expire' do + it 'expires the given key from the cache' do + cache.write(:foo, ['value']) + + expect(cache.read(:foo)).to contain_exactly('value') + expect(cache.expire(:foo)).to eq(1) + expect(cache.read(:foo)).to be_empty + end + end + + describe '#exist?' do + it 'checks whether the key exists' do + expect(cache.exist?(:foo)).to be(false) + + cache.write(:foo, ['value']) + + expect(cache.exist?(:foo)).to be(true) + end + end + + describe '#fetch' do + let(:blk) { -> { ['block value'] } } + + subject { cache.fetch(:foo, &blk) } + + it 'fetches the key from the cache when filled' do + cache.write(:foo, ['value']) + + is_expected.to contain_exactly('value') + end + + it 'writes the value of the provided block when empty' do + cache.expire(:foo) + + is_expected.to contain_exactly('block value') + expect(cache.read(:foo)).to contain_exactly('block value') + end + end + + describe '#include?' do + it 'checks inclusion in the Redis set' do + cache.write(:foo, ['value']) + + expect(cache.include?(:foo, 'value')).to be(true) + expect(cache.include?(:foo, 'bar')).to be(false) + end + end +end diff --git a/spec/lib/gitlab/utils/strong_memoize_spec.rb b/spec/lib/gitlab/utils/strong_memoize_spec.rb index 26baaf873a8..624e799c5e9 100644 --- a/spec/lib/gitlab/utils/strong_memoize_spec.rb +++ b/spec/lib/gitlab/utils/strong_memoize_spec.rb @@ -52,6 +52,22 @@ describe Gitlab::Utils::StrongMemoize do end end + describe '#strong_memoized?' do + let(:value) { :anything } + + subject { object.strong_memoized?(:method_name) } + + it 'returns false if the value is uncached' do + is_expected.to be(false) + end + + it 'returns true if the value is cached' do + object.method_name + + is_expected.to be(true) + end + end + describe '#clear_memoization' do let(:value) { 'mepmep' } |