Merge branch 'tc-realtime-every-pipeline-on-mr' into 'master'

Properly expire cache for **all** MRs of a pipeline

Closes #31040

See merge request !10770

7 files changed, 137 insertions, 88 deletions

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 75a8ab2f5cd..4be4aa9ffe2 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -84,24 +84,23 @@ module Ci
       end
 
       after_transition [:created, :pending] => :running do |pipeline|
-        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) }
+        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
 
       after_transition any => [:success] do |pipeline|
-        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(id) }
+        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
 
       after_transition [:created, :pending, :running] => :success do |pipeline|
-        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(id) }
+        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
       end
 
       after_transition do |pipeline, transition|
         next if transition.loopback?
 
         pipeline.run_after_commit do
-          PipelineHooksWorker.perform_async(id)
-          Ci::ExpirePipelineCacheService.new(project, nil)
-            .execute(pipeline)
+          PipelineHooksWorker.perform_async(pipeline.id)
+          ExpirePipelineCacheWorker.perform_async(pipeline.id)
         end
       end
 
@@ -389,6 +388,11 @@ module Ci
         .select { |merge_request| merge_request.head_pipeline.try(:id) == self.id }
     end
 
+    # All the merge requests for which the current pipeline runs/ran against
+    def all_merge_requests
+      @all_merge_requests ||= project.merge_requests.where(source_branch: ref)
+    end
+
     def detailed_status(current_user)
       Gitlab::Ci::Status::Pipeline::Factory
         .new(self, current_user)
diff --git a/app/services/ci/expire_pipeline_cache_service.rb b/app/services/ci/expire_pipeline_cache_service.rb
deleted file mode 100644
index 91d9c1d2ba1..00000000000
--- a/app/services/ci/expire_pipeline_cache_service.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-module Ci
-  class ExpirePipelineCacheService < BaseService
-    attr_reader :pipeline
-
-    def execute(pipeline)
-      @pipeline = pipeline
-      store = Gitlab::EtagCaching::Store.new
-
-      store.touch(project_pipelines_path)
-      store.touch(commit_pipelines_path) if pipeline.commit
-      store.touch(new_merge_request_pipelines_path)
-      merge_requests_pipelines_paths.each { |path| store.touch(path) }
-
-      Gitlab::Cache::Ci::ProjectPipelineStatus.update_for_pipeline(@pipeline)
-    end
-
-    private
-
-    def project_pipelines_path
-      Gitlab::Routing.url_helpers.namespace_project_pipelines_path(
-        project.namespace,
-        project,
-        format: :json)
-    end
-
-    def commit_pipelines_path
-      Gitlab::Routing.url_helpers.pipelines_namespace_project_commit_path(
-        project.namespace,
-        project,
-        pipeline.commit.id,
-        format: :json)
-    end
-
-    def new_merge_request_pipelines_path
-      Gitlab::Routing.url_helpers.new_namespace_project_merge_request_path(
-        project.namespace,
-        project,
-        format: :json)
-    end
-
-    def merge_requests_pipelines_paths
-      pipeline.merge_requests.collect do |merge_request|
-        Gitlab::Routing.url_helpers.pipelines_namespace_project_merge_request_path(
-          project.namespace,
-          project,
-          merge_request,
-          format: :json)
-      end
-    end
-  end
-end
diff --git a/app/workers/expire_pipeline_cache_worker.rb b/app/workers/expire_pipeline_cache_worker.rb
new file mode 100644
index 00000000000..603e2f1aaea
--- /dev/null
+++ b/app/workers/expire_pipeline_cache_worker.rb
@@ -0,0 +1,57 @@
+class ExpirePipelineCacheWorker
+  include Sidekiq::Worker
+  include PipelineQueue
+
+  def perform(pipeline_id)
+    pipeline = Ci::Pipeline.find_by(id: pipeline_id)
+    return unless pipeline
+
+    project = pipeline.project
+    store = Gitlab::EtagCaching::Store.new
+
+    store.touch(project_pipelines_path(project))
+    store.touch(commit_pipelines_path(project, pipeline.commit)) if pipeline.commit
+    store.touch(new_merge_request_pipelines_path(project))
+    each_pipelines_merge_request_path(project, pipeline) do |path|
+      store.touch(path)
+    end
+
+    Gitlab::Cache::Ci::ProjectPipelineStatus.update_for_pipeline(pipeline)
+  end
+
+  private
+
+  def project_pipelines_path(project)
+    Gitlab::Routing.url_helpers.namespace_project_pipelines_path(
+      project.namespace,
+      project,
+      format: :json)
+  end
+
+  def commit_pipelines_path(project, commit)
+    Gitlab::Routing.url_helpers.pipelines_namespace_project_commit_path(
+      project.namespace,
+      project,
+      commit.id,
+      format: :json)
+  end
+
+  def new_merge_request_pipelines_path(project)
+    Gitlab::Routing.url_helpers.new_namespace_project_merge_request_path(
+      project.namespace,
+      project,
+      format: :json)
+  end
+
+  def each_pipelines_merge_request_path(project, pipeline)
+    pipeline.all_merge_requests.each do |merge_request|
+      path = Gitlab::Routing.url_helpers.pipelines_namespace_project_merge_request_path(
+        project.namespace,
+        project,
+        merge_request,
+        format: :json)
+
+      yield(path)
+    end
+  end
+end
diff --git a/changelogs/unreleased/tc-realtime-every-pipeline-on-mr.yml b/changelogs/unreleased/tc-realtime-every-pipeline-on-mr.yml
new file mode 100644
index 00000000000..944baae257c
--- /dev/null
+++ b/changelogs/unreleased/tc-realtime-every-pipeline-on-mr.yml
@@ -0,0 +1,4 @@
+---
+title: Properly expire cache for all MRs of a pipeline
+merge_request: 10770
+author:
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index 675be77eafd..3b222ea1c3d 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -400,8 +400,8 @@ describe Ci::Pipeline, models: true do
     end
 
     describe 'pipeline caching' do
-      it 'executes ExpirePipelinesCacheService' do
-        expect_any_instance_of(Ci::ExpirePipelineCacheService).to receive(:execute).with(pipeline)
+      it 'performs ExpirePipelinesCacheWorker' do
+        expect(ExpirePipelineCacheWorker).to receive(:perform_async).with(pipeline.id)
 
         pipeline.cancel
       end
@@ -1039,11 +1039,12 @@ describe Ci::Pipeline, models: true do
   end
 
   describe "#merge_requests" do
-    let(:project) { create(:project, :repository) }
-    let(:pipeline) { FactoryGirl.create(:ci_empty_pipeline, status: 'created', project: project, ref: 'master', sha: project.repository.commit('master').id) }
+    let(:project) { create(:empty_project) }
+    let(:pipeline) { create(:ci_empty_pipeline, status: 'created', project: project, ref: 'master', sha: 'a288a022a53a5a944fae87bcec6efc87b7061808') }
 
     it "returns merge requests whose `diff_head_sha` matches the pipeline's SHA" do
       merge_request = create(:merge_request, source_project: project, source_branch: pipeline.ref)
+      allow_any_instance_of(MergeRequest).to receive(:diff_head_sha) { 'a288a022a53a5a944fae87bcec6efc87b7061808' }
 
       expect(pipeline.merge_requests).to eq([merge_request])
     end
@@ -1062,6 +1063,23 @@ describe Ci::Pipeline, models: true do
     end
   end
 
+  describe "#all_merge_requests" do
+    let(:project) { create(:empty_project) }
+    let(:pipeline) { create(:ci_empty_pipeline, status: 'created', project: project, ref: 'master') }
+
+    it "returns all merge requests having the same source branch" do
+      merge_request = create(:merge_request, source_project: project, source_branch: pipeline.ref)
+
+      expect(pipeline.all_merge_requests).to eq([merge_request])
+    end
+
+    it "doesn't return merge requests having a different source branch" do
+      create(:merge_request, source_project: project, source_branch: 'feature', target_branch: 'master')
+
+      expect(pipeline.all_merge_requests).to be_empty
+    end
+  end
+
   describe '#stuck?' do
     before do
       create(:ci_build, :pending, pipeline: pipeline)
diff --git a/spec/services/ci/expire_pipeline_cache_service_spec.rb b/spec/services/ci/expire_pipeline_cache_service_spec.rb
deleted file mode 100644
index 166c6dfc93e..00000000000
--- a/spec/services/ci/expire_pipeline_cache_service_spec.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-require 'spec_helper'
-
-describe Ci::ExpirePipelineCacheService, services: true do
-  let(:user) { create(:user) }
-  let(:project) { create(:empty_project) }
-  let(:pipeline) { create(:ci_pipeline, project: project) }
-  subject { described_class.new(project, user) }
-
-  describe '#execute' do
-    it 'invalidate Etag caching for project pipelines path' do
-      pipelines_path = "/#{project.full_path}/pipelines.json"
-      new_mr_pipelines_path = "/#{project.full_path}/merge_requests/new.json"
-
-      expect_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch).with(pipelines_path)
-      expect_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch).with(new_mr_pipelines_path)
-
-      subject.execute(pipeline)
-    end
-
-    it 'updates the cached status for a project' do
-      expect(Gitlab::Cache::Ci::ProjectPipelineStatus).to receive(:update_for_pipeline).
-                                                            with(pipeline)
-
-      subject.execute(pipeline)
-    end
-  end
-end
diff --git a/spec/workers/expire_pipeline_cache_worker_spec.rb b/spec/workers/expire_pipeline_cache_worker_spec.rb
new file mode 100644
index 00000000000..ceba604dea2
--- /dev/null
+++ b/spec/workers/expire_pipeline_cache_worker_spec.rb
@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe ExpirePipelineCacheWorker do
+  let(:user) { create(:user) }
+  let(:project) { create(:empty_project) }
+  let(:pipeline) { create(:ci_pipeline, project: project) }
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'invalidates Etag caching for project pipelines path' do
+      pipelines_path = "/#{project.full_path}/pipelines.json"
+      new_mr_pipelines_path = "/#{project.full_path}/merge_requests/new.json"
+
+      expect_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch).with(pipelines_path)
+      expect_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch).with(new_mr_pipelines_path)
+
+      subject.perform(pipeline.id)
+    end
+
+    it 'invalidates Etag caching for merge request pipelines if pipeline runs on any commit of that source branch' do
+      pipeline = create(:ci_empty_pipeline, status: 'created', project: project, ref: 'master')
+      merge_request = create(:merge_request, source_project: project, source_branch: pipeline.ref)
+      merge_request_pipelines_path = "/#{project.full_path}/merge_requests/#{merge_request.iid}/pipelines.json"
+
+      allow_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch)
+      expect_any_instance_of(Gitlab::EtagCaching::Store).to receive(:touch).with(merge_request_pipelines_path)
+
+      subject.perform(pipeline.id)
+    end
+
+    it "doesn't do anything if the pipeline not exist" do
+      expect_any_instance_of(Gitlab::EtagCaching::Store).not_to receive(:touch)
+
+      subject.perform(617748)
+    end
+
+    it 'updates the cached status for a project' do
+      expect(Gitlab::Cache::Ci::ProjectPipelineStatus).to receive(:update_for_pipeline).
+                                                            with(pipeline)
+
+      subject.perform(pipeline.id)
+    end
+  end
+end