Refactor cache refreshing/expiring

This refactors repository caching so it's possible to selectively
refresh certain caches, instead of just expiring and refreshing
everything.

To allow this the various methods that were cached (e.g. "tag_count" and
"readme") use a similar pattern that makes expiring and refreshing
their data much easier.

In this new setup caches are refreshed as follows:

1. After a commit (but before running ProjectCacheWorker) we expire some
   basic caches such as the commit count and repository size.

2. ProjectCacheWorker will recalculate the commit count, repository
   size, then refresh a specific set of caches based on the list of
   files changed in a push payload.

This requires a bunch of changes to the various methods that may be
cached. For one, data should not be cached if a branch used or the
entire repository does not exist. To prevent all these methods from
handling this manually this is taken care of in
Repository#cache_method_output. Some methods still manually check for
the existence of a repository but this result is also cached.

With selective flushing implemented ProjectCacheWorker no longer uses an
exclusive lease for all of its work. Instead this worker only uses a
lease to limit the number of times the repository size is updated as
this is a fairly expensive operation.

1 files changed, 51 insertions, 35 deletions

diff --git a/spec/workers/project_cache_worker_spec.rb b/spec/workers/project_cache_worker_spec.rb
index bfa8c0ff2c6..855c28b584e 100644
--- a/spec/workers/project_cache_worker_spec.rb
+++ b/spec/workers/project_cache_worker_spec.rb
@@ -2,62 +2,78 @@ require 'spec_helper'
 
 describe ProjectCacheWorker do
   let(:project) { create(:project) }
+  let(:worker) { described_class.new }
 
-  subject { described_class.new }
-
-  describe '.perform_async' do
-    it 'schedules the job when no lease exists' do
-      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:exists?).
-        and_return(false)
+  describe '#perform' do
+    before do
+      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).
+        and_return(true)
+    end
 
-      expect_any_instance_of(described_class).to receive(:perform)
+    context 'with a non-existing project' do
+      it 'does nothing' do
+        expect(worker).not_to receive(:update_repository_size)
 
-      described_class.perform_async(project.id)
+        worker.perform(-1)
+      end
     end
 
-    it 'does not schedule the job when a lease exists' do
-      allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:exists?).
-        and_return(true)
+    context 'with an existing project without a repository' do
+      it 'does nothing' do
+        allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
 
-      expect_any_instance_of(described_class).not_to receive(:perform)
+        expect(worker).not_to receive(:update_repository_size)
 
-      described_class.perform_async(project.id)
+        worker.perform(project.id)
+      end
     end
-  end
 
-  describe '#perform' do
-    context 'when an exclusive lease can be obtained' do
-      before do
-        allow(subject).to receive(:try_obtain_lease_for).with(project.id).
-          and_return(true)
-      end
+    context 'with an existing project' do
+      it 'updates the repository size' do
+        expect(worker).to receive(:update_repository_size).and_call_original
 
-      it 'updates project cache data' do
-        expect_any_instance_of(Repository).to receive(:size)
-        expect_any_instance_of(Repository).to receive(:commit_count)
+        worker.perform(project.id)
+      end
 
-        expect_any_instance_of(Project).to receive(:update_repository_size)
-        expect_any_instance_of(Project).to receive(:update_commit_count)
+      it 'updates the commit count' do
+        expect_any_instance_of(Project).to receive(:update_commit_count).
+          and_call_original
 
-        subject.perform(project.id)
+        worker.perform(project.id)
       end
 
-      it 'handles missing repository data' do
-        expect_any_instance_of(Repository).to receive(:exists?).and_return(false)
-        expect_any_instance_of(Repository).not_to receive(:size)
+      it 'refreshes the method caches' do
+        expect_any_instance_of(Repository).to receive(:refresh_method_caches).
+          with(%i(readme)).
+          and_call_original
 
-        subject.perform(project.id)
+        worker.perform(project.id, %i(readme))
       end
     end
+  end
 
-    context 'when an exclusive lease can not be obtained' do
-      it 'does nothing' do
-        allow(subject).to receive(:try_obtain_lease_for).with(project.id).
+  describe '#update_repository_size' do
+    context 'when a lease could not be obtained' do
+      it 'does not update the repository size' do
+        allow(worker).to receive(:try_obtain_lease_for).
+          with(project.id, :update_repository_size).
           and_return(false)
 
-        expect(subject).not_to receive(:update_caches)
+        expect(project).not_to receive(:update_repository_size)
+
+        worker.update_repository_size(project)
+      end
+    end
+
+    context 'when a lease could be obtained' do
+      it 'updates the repository size' do
+        allow(worker).to receive(:try_obtain_lease_for).
+          with(project.id, :update_repository_size).
+          and_return(true)
+
+        expect(project).to receive(:update_repository_size).and_call_original
 
-        subject.perform(project.id)
+        worker.update_repository_size(project)
       end
     end
   end