diff options
author | Stan Hu <stanhu@gmail.com> | 2019-08-16 19:53:56 +0000 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2019-08-16 19:53:56 +0000 |
commit | f14647fdae4a07c3c59665576b70f847ab866c58 (patch) | |
tree | 3dbeb5f11039722f520694041e6e161d15b628f0 /spec/workers | |
parent | b3145bc976836f4c28a2f97a57e40b9f315aa3a4 (diff) | |
download | gitlab-ce-f14647fdae4a07c3c59665576b70f847ab866c58.tar.gz |
Expire project caches once per push instead of once per ref
Previously `ProjectCacheWorker` would be scheduled once per ref, which
would generate unnecessary I/O and load on Sidekiq, especially if many
tags or branches were pushed at once. `ProjectCacheWorker` would expire
three items:
1. Repository size: This only needs to be updated once per push.
2. Commit count: This only needs to be updated if the default branch
is updated.
3. Project method caches: This only needs to be updated if the default
branch changes, but only if certain files change (e.g. README,
CHANGELOG, etc.).
Because the third item requires looking at the actual changes in the
commit deltas, we schedule one `ProjectCacheWorker` to handle the first
two cases, and schedule a separate `ProjectCacheWorker` for the third
case if it is needed. As a result, this brings down the number of
`ProjectCacheWorker` jobs from N to 2.
Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/52046
Diffstat (limited to 'spec/workers')
-rw-r--r-- | spec/workers/post_receive_spec.rb | 71 | ||||
-rw-r--r-- | spec/workers/project_cache_worker_spec.rb | 10 |
2 files changed, 76 insertions, 5 deletions
diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb index 3b69b81f12e..c8a0c22b0e8 100644 --- a/spec/workers/post_receive_spec.rb +++ b/spec/workers/post_receive_spec.rb @@ -37,6 +37,29 @@ describe PostReceive do end describe "#process_project_changes" do + context 'with an empty project' do + let(:empty_project) { create(:project, :empty_repo) } + let(:changes) { "123456 789012 refs/heads/tést1\n" } + + before do + allow_any_instance_of(Gitlab::GitPostReceive).to receive(:identify).and_return(empty_project.owner) + allow(Gitlab::GlRepository).to receive(:parse).and_return([empty_project, Gitlab::GlRepository::PROJECT]) + end + + it 'expire the status cache' do + expect(empty_project.repository).to receive(:expire_status_cache) + + perform + end + + it 'schedules a cache update for commit count and size' do + expect(ProjectCacheWorker).to receive(:perform_async) + .with(empty_project.id, [], [:repository_size, :commit_count], true) + + perform + end + end + context 'empty changes' do it "does not call any PushService but runs after project hooks" do expect(Git::BranchPushService).not_to receive(:new) @@ -67,15 +90,22 @@ describe PostReceive do context "branches" do let(:changes) do <<~EOF - '123456 789012 refs/heads/tést1' - '123456 789012 refs/heads/tést2' + 123456 789012 refs/heads/tést1 + 123456 789012 refs/heads/tést2 EOF end it 'expires the branches cache' do expect(project.repository).to receive(:expire_branches_cache).once - described_class.new.perform(gl_repository, key_id, base64_changes) + perform + end + + it 'expires the status cache' do + expect(project).to receive(:empty_repo?).and_return(true) + expect(project.repository).to receive(:expire_status_cache) + + perform end it 'calls Git::BranchPushService' do @@ -87,6 +117,30 @@ describe PostReceive do perform end + + it 'schedules a cache update for repository size only' do + expect(ProjectCacheWorker).to receive(:perform_async) + .with(project.id, [], [:repository_size], true) + + perform + end + + context 'with a default branch' do + let(:changes) do + <<~EOF + 123456 789012 refs/heads/tést1 + 123456 789012 refs/heads/tést2 + 678912 123455 refs/heads/#{project.default_branch} + EOF + end + + it 'schedules a cache update for commit count and size' do + expect(ProjectCacheWorker).to receive(:perform_async) + .with(project.id, [], [:repository_size, :commit_count], true) + + perform + end + end end context "tags" do @@ -107,7 +161,7 @@ describe PostReceive do it 'does not expire branches cache' do expect(project.repository).not_to receive(:expire_branches_cache) - described_class.new.perform(gl_repository, key_id, base64_changes) + perform end it "only invalidates tags once" do @@ -115,7 +169,7 @@ describe PostReceive do expect(project.repository).to receive(:expire_caches_for_tags).once.and_call_original expect(project.repository).to receive(:expire_tags_cache).once.and_call_original - described_class.new.perform(gl_repository, key_id, base64_changes) + perform end it "calls Git::TagPushService" do @@ -129,6 +183,13 @@ describe PostReceive do perform end + + it 'schedules a single ProjectCacheWorker update' do + expect(ProjectCacheWorker).to receive(:perform_async) + .with(project.id, [], [:repository_size], true) + + perform + end end context "merge-requests" do diff --git a/spec/workers/project_cache_worker_spec.rb b/spec/workers/project_cache_worker_spec.rb index edc55920b8e..7f3c4881b89 100644 --- a/spec/workers/project_cache_worker_spec.rb +++ b/spec/workers/project_cache_worker_spec.rb @@ -49,6 +49,16 @@ describe ProjectCacheWorker do worker.perform(project.id, %w(readme)) end + context 'with statistics disabled' do + let(:statistics) { [] } + + it 'does not update the project statistics' do + expect(worker).not_to receive(:update_statistics) + + worker.perform(project.id, [], [], false) + end + end + context 'with statistics' do let(:statistics) { %w(repository_size) } |