Pre-create all builds for Pipeline when a trigger is received

This change simplifies a Pipeline processing by introducing a special new status: created.
This status is used for all builds that are created for a pipeline.
We are then processing next stages and queueing some of the builds (created -> pending) or skipping them (created -> skipped).
This makes it possible to simplify and solve a few ordering problems with how previously builds were scheduled.
This also allows us to visualise a full pipeline (with created builds).

This also removes an after_touch used for updating a pipeline state parameters.
Right now in various places we explicitly call a reload_status! on pipeline to force it to be updated and saved.

3 files changed, 63 insertions, 303 deletions

diff --git a/spec/models/build_spec.rb b/spec/models/build_spec.rb
index 9ecc9aac84b..60a221eba50 100644
--- a/spec/models/build_spec.rb
+++ b/spec/models/build_spec.rb
@@ -764,6 +764,53 @@ describe Ci::Build, models: true do
     end
   end
 
+  describe '#when' do
+    subject { build.when }
+
+    context 'if is undefined' do
+      before do
+        build.when = nil
+      end
+
+      context 'use from gitlab-ci.yml' do
+        before do
+          stub_ci_pipeline_yaml_file(config)
+        end
+
+        context 'if config is not found' do
+          let(:config) { nil }
+
+          it { is_expected.to eq('on_success') }
+        end
+
+        context 'if config does not have a questioned job' do
+          let(:config) do
+            YAML.dump({
+                        test_other: {
+                          script: 'Hello World'
+                        }
+                      })
+          end
+
+          it { is_expected.to eq('on_success') }
+        end
+
+        context 'if config has when' do
+          let(:config) do
+            YAML.dump({
+                        test: {
+                          script: 'Hello World',
+                          when: 'always'
+                        }
+                      })
+          end
+
+          it { is_expected.to eq('always') }
+        end
+      end
+    end
+  end
+
   describe '#retryable?' do
     context 'when build is running' do
       before do
diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb
index ccee591cf7a..fdb579ab45c 100644
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@@ -38,9 +38,6 @@ describe Ci::Pipeline, models: true do
     it { expect(pipeline.sha).to start_with(subject) }
   end
 
-  describe '#create_next_builds' do
-  end
-
   describe '#retried' do
     subject { pipeline.retried }
 
@@ -54,304 +51,20 @@ describe Ci::Pipeline, models: true do
     end
   end
 
-  describe '#create_builds' do
-    let!(:pipeline) { FactoryGirl.create :ci_pipeline, project: project, ref: 'master', tag: false }
-
-    def create_builds(trigger_request = nil)
-      pipeline.create_builds(nil, trigger_request)
-    end
-
-    def create_next_builds
-      pipeline.create_next_builds(pipeline.builds.order(:id).last)
-    end
-
-    it 'creates builds' do
-      expect(create_builds).to be_truthy
-      pipeline.builds.update_all(status: "success")
-      expect(pipeline.builds.count(:all)).to eq(2)
-
-      expect(create_next_builds).to be_truthy
-      pipeline.builds.update_all(status: "success")
-      expect(pipeline.builds.count(:all)).to eq(4)
-
-      expect(create_next_builds).to be_truthy
-      pipeline.builds.update_all(status: "success")
-      expect(pipeline.builds.count(:all)).to eq(5)
-
-      expect(create_next_builds).to be_falsey
-    end
-
-    context 'custom stage with first job allowed to fail' do
-      let(:yaml) do
-        {
-          stages: ['clean', 'test'],
-          clean_job: {
-            stage: 'clean',
-            allow_failure: true,
-            script: 'BUILD',
-          },
-          test_job: {
-            stage: 'test',
-            script: 'TEST',
-          },
-        }
-      end
-
-      before do
-        stub_ci_pipeline_yaml_file(YAML.dump(yaml))
-        create_builds
-      end
-
-      it 'properly schedules builds' do
-        expect(pipeline.builds.pluck(:status)).to contain_exactly('pending')
-        pipeline.builds.running_or_pending.each(&:drop)
-        expect(pipeline.builds.pluck(:status)).to contain_exactly('pending', 'failed')
-      end
-    end
-
-    context 'properly creates builds when "when" is defined' do
-      let(:yaml) do
-        {
-          stages: ["build", "test", "test_failure", "deploy", "cleanup"],
-          build: {
-            stage: "build",
-            script: "BUILD",
-          },
-          test: {
-            stage: "test",
-            script: "TEST",
-          },
-          test_failure: {
-            stage: "test_failure",
-            script: "ON test failure",
-            when: "on_failure",
-          },
-          deploy: {
-            stage: "deploy",
-            script: "PUBLISH",
-          },
-          cleanup: {
-            stage: "cleanup",
-            script: "TIDY UP",
-            when: "always",
-          }
-        }
-      end
-
-      before do
-        stub_ci_pipeline_yaml_file(YAML.dump(yaml))
-      end
-
-      context 'when builds are successful' do
-        it 'properly creates builds' do
-          expect(create_builds).to be_truthy
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy', 'cleanup')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'success', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'success', 'success', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('success')
-        end
-      end
-
-      context 'when test job fails' do
-        it 'properly creates builds' do
-          expect(create_builds).to be_truthy
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'failed', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'failed', 'success', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('failed')
-        end
-      end
-
-      context 'when test and test_failure jobs fail' do
-        it 'properly creates builds' do
-          expect(create_builds).to be_truthy
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'failed', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'test_failure', 'cleanup')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'failed', 'failed', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('failed')
-        end
-      end
-
-      context 'when deploy job fails' do
-        it 'properly creates builds' do
-          expect(create_builds).to be_truthy
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'success', 'pending')
-          pipeline.builds.running_or_pending.each(&:drop)
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test', 'deploy', 'cleanup')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'success', 'failed', 'pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'success', 'failed', 'success')
-          pipeline.reload
-          expect(pipeline.status).to eq('failed')
-        end
-      end
-
-      context 'when build is canceled in the second stage' do
-        it 'does not schedule builds after build has been canceled' do
-          expect(create_builds).to be_truthy
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('pending')
-          pipeline.builds.running_or_pending.each(&:success)
-
-          expect(pipeline.builds.running_or_pending).not_to be_empty
-
-          expect(pipeline.builds.pluck(:name)).to contain_exactly('build', 'test')
-          expect(pipeline.builds.pluck(:status)).to contain_exactly('success', 'pending')
-          pipeline.builds.running_or_pending.each(&:cancel)
-
-          expect(pipeline.builds.running_or_pending).to be_empty
-          expect(pipeline.reload.status).to eq('canceled')
-        end
-      end
-
-      context 'when listing manual actions' do
-        let(:yaml) do
-          {
-            stages: ["build", "test", "staging", "production", "cleanup"],
-            build: {
-              stage: "build",
-              script: "BUILD",
-            },
-            test: {
-              stage: "test",
-              script: "TEST",
-            },
-            staging: {
-              stage: "staging",
-              script: "PUBLISH",
-            },
-            production: {
-              stage: "production",
-              script: "PUBLISH",
-              when: "manual",
-            },
-            cleanup: {
-              stage: "cleanup",
-              script: "TIDY UP",
-              when: "always",
-            },
-            clear_cache: {
-              stage: "cleanup",
-              script: "CLEAR CACHE",
-              when: "manual",
-            }
-          }
-        end
-
-        it 'returns only for skipped builds' do
-          # currently all builds are created
-          expect(create_builds).to be_truthy
-          expect(manual_actions).to be_empty
-
-          # succeed stage build
-          pipeline.builds.running_or_pending.each(&:success)
-          expect(manual_actions).to be_empty
-
-          # succeed stage test
-          pipeline.builds.running_or_pending.each(&:success)
-          expect(manual_actions).to be_empty
-
-          # succeed stage staging and skip stage production
-          pipeline.builds.running_or_pending.each(&:success)
-          expect(manual_actions).to be_many # production and clear cache
-
-          # succeed stage cleanup
-          pipeline.builds.running_or_pending.each(&:success)
-
-          # after processing a pipeline we should have 6 builds, 5 succeeded
-          expect(pipeline.builds.count).to eq(6)
-          expect(pipeline.builds.success.count).to eq(4)
-        end
-
-        def manual_actions
-          pipeline.manual_actions
-        end
-      end
-    end
-
-    context 'when no builds created' do
-      let(:pipeline) { build(:ci_pipeline) }
-
-      before do
-        stub_ci_pipeline_yaml_file(YAML.dump(before_script: ['ls']))
-      end
-
-      it 'returns false' do
-        expect(pipeline.create_builds(nil)).to be_falsey
-        expect(pipeline).not_to be_persisted
-      end
-    end
-  end
-
   describe "#finished_at" do
     let(:pipeline) { FactoryGirl.create :ci_pipeline }
 
     it "returns finished_at of latest build" do
       build = FactoryGirl.create :ci_build, pipeline: pipeline, finished_at: Time.now - 60
       FactoryGirl.create :ci_build, pipeline: pipeline, finished_at: Time.now - 120
+      pipeline.reload_status!
 
       expect(pipeline.finished_at.to_i).to eq(build.finished_at.to_i)
     end
 
     it "returns nil if there is no finished build" do
       FactoryGirl.create :ci_not_started_build, pipeline: pipeline
+      pipeline.reload_status!
 
       expect(pipeline.finished_at).to be_nil
     end
@@ -359,7 +72,7 @@ describe Ci::Pipeline, models: true do
 
   describe "coverage" do
     let(:project) { FactoryGirl.create :empty_project, build_coverage_regex: "/.*/" }
-    let(:pipeline) { FactoryGirl.create :ci_pipeline, project: project }
+    let(:pipeline) { FactoryGirl.create :ci_empty_pipeline, project: project }
 
     it "calculates average when there are two builds with coverage" do
       FactoryGirl.create :ci_build, name: "rspec", coverage: 30, pipeline: pipeline
@@ -426,31 +139,30 @@ describe Ci::Pipeline, models: true do
     end
   end
 
-  describe '#update_state' do
-    it 'executes update_state after touching object' do
-      expect(pipeline).to receive(:update_state).and_return(true)
-      pipeline.touch
-    end
+  describe '#reload_status!' do
+    let(:pipeline) { create :ci_empty_pipeline, project: project }
 
     context 'dependent objects' do
-      let(:commit_status) { build :commit_status, pipeline: pipeline }
+      let(:commit_status) { create :commit_status, :pending, pipeline: pipeline }
+
+      it 'executes reload_status! after succeeding dependent object' do
+        expect(pipeline).to receive(:reload_status!).and_return(true)
 
-      it 'executes update_state after saving dependent object' do
-        expect(pipeline).to receive(:update_state).and_return(true)
-        commit_status.save
+        commit_status.success
       end
     end
 
-    context 'update state' do
+    context 'updates' do
       let(:current) { Time.now.change(usec: 0) }
-      let(:build) { FactoryGirl.create :ci_build, :success, pipeline: pipeline, started_at: current - 120, finished_at: current - 60 }
+      let(:build) { FactoryGirl.create :ci_build, pipeline: pipeline, started_at: current - 120, finished_at: current - 60 }
 
       before do
         build
+        pipeline.reload_status!
       end
 
       [:status, :started_at, :finished_at, :duration].each do |param|
-        it "update #{param}" do
+        it "#{param}" do
           expect(pipeline.send(param)).to eq(build.send(param))
         end
       end
diff --git a/spec/models/project_services/hipchat_service_spec.rb b/spec/models/project_services/hipchat_service_spec.rb
index bf438b26690..1b383219eb9 100644
--- a/spec/models/project_services/hipchat_service_spec.rb
+++ b/spec/models/project_services/hipchat_service_spec.rb
@@ -291,7 +291,8 @@ describe HipchatService, models: true do
     end
 
     context 'build events' do
-      let(:build) { create(:ci_build) }
+      let(:pipeline) { create(:ci_empty_pipeline) }
+      let(:build) { create(:ci_build, pipeline: pipeline) }
       let(:data) { Gitlab::BuildDataBuilder.build(build) }
 
       context 'for failed' do