Add Etag caching on individual pipeline requests

Also, this commit adds a changelog entry and some misc changes which I
didn't think were big enough to give their own commit.

8 files changed, 85 insertions, 11 deletions

diff --git a/app/controllers/projects/pipelines_controller.rb b/app/controllers/projects/pipelines_controller.rb
index 148614cbfb8..ea2915a4de6 100644
--- a/app/controllers/projects/pipelines_controller.rb
+++ b/app/controllers/projects/pipelines_controller.rb
@@ -8,6 +8,8 @@ class Projects::PipelinesController < Projects::ApplicationController
 
   wrap_parameters Ci::Pipeline
 
+  POLLING_INTERVAL = 10_000
+
   def index
     @scope = params[:scope]
     @pipelines = PipelinesFinder
@@ -31,7 +33,7 @@ class Projects::PipelinesController < Projects::ApplicationController
     respond_to do |format|
       format.html
       format.json do
-        Gitlab::PollingInterval.set_header(response, interval: 10_000)
+        Gitlab::PollingInterval.set_header(response, interval: POLLING_INTERVAL)
 
         render json: {
           pipelines: PipelineSerializer
@@ -69,9 +71,11 @@ class Projects::PipelinesController < Projects::ApplicationController
     respond_to do |format|
       format.html
       format.json do
+        Gitlab::PollingInterval.set_header(response, interval: POLLING_INTERVAL)
+
         render json: PipelineSerializer.
           new(project: @project, user: @current_user).
-          represent(@pipeline, with_jobs: true)
+          represent(@pipeline)
       end
     end
   end
diff --git a/app/serializers/pipeline_entity.rb b/app/serializers/pipeline_entity.rb
index 2565ade7307..d221b09d9cb 100644
--- a/app/serializers/pipeline_entity.rb
+++ b/app/serializers/pipeline_entity.rb
@@ -11,13 +11,12 @@ class PipelineEntity < Grape::Entity
       pipeline)
   end
 
-  expose :builds, as: :jobs, if: lambda { |_, opts| opts[:with_jobs] }, with: BuildEntity
+  expose :stages, with: StageEntity
 
   expose :details do
     expose :detailed_status, as: :status, with: StatusEntity
     expose :duration
     expose :finished_at
-    expose :stages, using: StageEntity
     expose :artifacts, using: BuildArtifactEntity
     expose :manual_actions, using: BuildActionEntity
   end
diff --git a/app/serializers/stage_entity.rb b/app/serializers/stage_entity.rb
index 7a047bdc712..5018966b248 100644
--- a/app/serializers/stage_entity.rb
+++ b/app/serializers/stage_entity.rb
@@ -7,9 +7,7 @@ class StageEntity < Grape::Entity
     "#{stage.name}: #{detailed_status.label}"
   end
 
-  expose :detailed_status,
-    as: :status,
-    with: StatusEntity
+  expose :detailed_status, as: :status, with: StatusEntity
 
   expose :path do |stage|
     namespace_project_pipeline_path(
@@ -28,6 +26,8 @@ class StageEntity < Grape::Entity
       format: :json)
   end
 
+  expose :builds, as: :jobs, using: BuildEntity
+
   private
 
   alias_method :stage, :object
diff --git a/app/services/ci/expire_pipeline_cache_service.rb b/app/services/ci/expire_pipeline_cache_service.rb
new file mode 100644
index 00000000000..ad968085aa7
--- /dev/null
+++ b/app/services/ci/expire_pipeline_cache_service.rb
@@ -0,0 +1,60 @@
+module Ci
+  class ExpirePipelineCacheService < BaseService
+    attr_reader :pipeline
+
+    def execute(pipeline)
+      @pipeline = pipeline
+      store = Gitlab::EtagCaching::Store.new
+
+      store.touch(project_pipeline_path)
+      store.touch(project_pipelines_path)
+      store.touch(commit_pipelines_path) if pipeline.commit
+      store.touch(new_merge_request_pipelines_path)
+      merge_requests_pipelines_paths.each { |path| store.touch(path) }
+
+      Gitlab::Cache::Ci::ProjectPipelineStatus.update_for_pipeline(@pipeline)
+    end
+
+    private
+
+    def project_pipelines_path
+      Gitlab::Routing.url_helpers.namespace_project_pipelines_path(
+        project.namespace,
+        project,
+        format: :json)
+    end
+
+    def commit_pipelines_path
+      Gitlab::Routing.url_helpers.pipelines_namespace_project_commit_path(
+        project.namespace,
+        project,
+        pipeline.commit.id,
+        format: :json)
+    end
+
+    def new_merge_request_pipelines_path
+      Gitlab::Routing.url_helpers.new_namespace_project_merge_request_path(
+        project.namespace,
+        project,
+        format: :json)
+    end
+
+    def merge_requests_pipelines_paths
+      pipeline.merge_requests.collect do |merge_request|
+        Gitlab::Routing.url_helpers.pipelines_namespace_project_merge_request_path(
+          project.namespace,
+          project,
+          merge_request,
+          format: :json)
+      end
+    end
+
+    def project_pipeline_path
+      Gitlab::Routing.url_helpers.namespace_project_pipeline_path(
+        project.namespace,
+        project,
+        pipeline,
+        format: :json)
+    end
+  end
+end
diff --git a/changelogs/unreleased/zj-real-time-pipelines.yml b/changelogs/unreleased/zj-real-time-pipelines.yml
new file mode 100644
index 00000000000..eec22e67467
--- /dev/null
+++ b/changelogs/unreleased/zj-real-time-pipelines.yml
@@ -0,0 +1,4 @@
+---
+title: Pipeline view updates in near real time
+merge_request: 10777
+author:
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index aac210f19e8..692c909d838 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -36,7 +36,11 @@ module Gitlab
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z),
           'project_pipelines'
-        )
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z),
+          'project_pipeline'
+        ),
       ].freeze
 
       def self.match(env)
diff --git a/spec/controllers/projects/pipelines_controller_spec.rb b/spec/controllers/projects/pipelines_controller_spec.rb
index 592b976888a..670052f079a 100644
--- a/spec/controllers/projects/pipelines_controller_spec.rb
+++ b/spec/controllers/projects/pipelines_controller_spec.rb
@@ -1,6 +1,8 @@
 require 'spec_helper'
 
 describe Projects::PipelinesController do
+  include ApiHelpers
+
   let(:user) { create(:user) }
   let(:project) { create(:empty_project, :public) }
 
@@ -31,6 +33,7 @@ describe Projects::PipelinesController do
       expect(json_response['count']['running']).to eq 1
       expect(json_response['count']['pending']).to eq 1
       expect(json_response['count']['finished']).to eq 1
+      expect(json_response).not_to have_key('stages')
     end
   end
 
@@ -43,7 +46,7 @@ describe Projects::PipelinesController do
       expect(response).to have_http_status(:ok)
       expect(json_response).not_to be_an(Array)
       expect(json_response['id']).to be(pipeline.id)
-      expect(json_response).to have_key('jobs')
+      expect(json_response).to have_key('stages')
     end
   end
 
diff --git a/spec/serializers/pipeline_entity_spec.rb b/spec/serializers/pipeline_entity_spec.rb
index 93d5a21419d..8256224652a 100644
--- a/spec/serializers/pipeline_entity_spec.rb
+++ b/spec/serializers/pipeline_entity_spec.rb
@@ -20,7 +20,7 @@ describe PipelineEntity do
 
       it 'contains required fields' do
         expect(subject).to include :id, :user, :path
-        expect(subject).to include :ref, :commit
+        expect(subject).to include :ref, :commit, :stages
         expect(subject).to include :updated_at, :created_at
       end
 
@@ -29,7 +29,7 @@ describe PipelineEntity do
         expect(subject[:details])
           .to include :duration, :finished_at
         expect(subject[:details])
-          .to include :stages, :artifacts, :manual_actions
+          .to include :artifacts, :manual_actions
         expect(subject[:details][:status]).to include :icon, :favicon, :text, :label
       end