Merge branch 'build-types' into 'master'

Allow to specify flexible list of types in yaml

First part of flexible pipeline build in GitLab CI

Having following `.gitlab-ci.yml`:
```
types:
- test
- deploy
- notify

rspec:
  script: "rspec"

rubocop:
  script: "rubocop"

staging:
  type: deploy
  script: "echo deploy"
  only:
  - master

production:
  type: deploy
  script: "echo production"
  only:
  - tags

dockerhub:
  type: notify
  script: "curl http://docker/hub/web/hook"

downstream:
  type: notify
  script: "curl http://build/downstream/jobs"
```

GitLab CI will trigger two test jobs in parallel, when finished it will trigged either staging or production, when finished it will trigger dockerhub and downstream in parallel.

The UI (screenshots are not for above YAML):
![Screen_Shot_2015-07-10_at_15.56.26](https://gitlab.com/gitlab-org/gitlab-ci/uploads/1f714b73772cf0d44168fb8e20e35561/Screen_Shot_2015-07-10_at_15.56.26.png)
![Screen_Shot_2015-07-10_at_15.57.19](https://gitlab.com/gitlab-org/gitlab-ci/uploads/fc9f458f2ca517d923a4382466fa99eb/Screen_Shot_2015-07-10_at_15.57.19.png)

TODO:
- [x] Implement in CI
- [x] Specs
- [x] Changelog
- [x] CI tests
- [ ] Documentation

/cc @vsizov @sytses @dzaporozhets

See merge request !198

2 files changed, 50 insertions, 38 deletions

diff --git a/app/models/build.rb b/app/models/build.rb
index 5e6522d..11bd967 100644
--- a/app/models/build.rb
+++ b/app/models/build.rb
@@ -111,8 +111,8 @@ class Build < ActiveRecord::Base
         WebHookService.new.build_end(build)
       end
 
-      if build.commit.success? && !build.deploy?
-        build.commit.create_deploy_builds
+      if build.commit.success?
+        build.commit.create_next_builds
       end
 
       project.execute_services(build)
diff --git a/app/models/commit.rb b/app/models/commit.rb
index e173ed4..eea3653 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -93,30 +93,50 @@ class Commit < ActiveRecord::Base
     recipients.uniq
   end
 
-  def create_builds
-    return if skip_ci?
+  def job_type
+    return unless config_processor
+    job_types = builds_without_retry.select(&:active?).map(&:job_type)
+    config_processor.types.find { |job_type| job_types.include? job_type }
+  end
 
-    begin
-      builds_for_ref = config_processor.builds_for_ref(ref, tag)
-    rescue GitlabCiYamlProcessor::ValidationError => e
-      save_yaml_error(e.message) and return
-    rescue Exception => e
-      logger.error e.message + "\n" + e.backtrace.join("\n")
-      save_yaml_error("Undefined yaml error") and return
-    end
+  def create_builds_for_type(job_type)
+    return if skip_ci?
+    return unless config_processor
 
-    builds_for_ref.each do |build_attrs|
+    builds_attrs = config_processor.builds_for_type_and_ref(job_type, ref, tag)
+    builds_attrs.map do |build_attrs|
       builds.create!({
         project: project,
         name: build_attrs[:name],
         commands: build_attrs[:script],
         tag_list: build_attrs[:tags],
         options: build_attrs[:options],
-        allow_failure: build_attrs[:allow_failure]
+        allow_failure: build_attrs[:allow_failure],
+        job_type: build_attrs[:type]
       })
     end
   end
 
+  def create_next_builds
+    return if skip_ci?
+    return unless config_processor
+
+    build_types = builds.group_by(&:job_type)
+
+    config_processor.types.any? do |job_type|
+      !build_types.include?(job_type) && create_builds_for_type(job_type).present?
+    end
+  end
+
+  def create_builds
+    return if skip_ci?
+    return unless config_processor
+
+    config_processor.types.any? do |job_type|
+      create_builds_for_type(job_type).present?
+    end
+  end
+
   def builds_without_retry
     @builds_without_retry ||=
       begin
@@ -127,33 +147,17 @@ class Commit < ActiveRecord::Base
       end
   end
 
-  def retried_builds
-    @retried_builds ||= (builds - builds_without_retry)
-  end
+  def builds_without_retry_sorted
+    return builds_without_retry unless config_processor
 
-  def create_deploy_builds
-    return if skip_ci?
-
-    begin
-      deploy_builds_for_ref = config_processor.deploy_builds_for_ref(ref, tag)
-    rescue GitlabCiYamlProcessor::ValidationError => e
-      save_yaml_error(e.message) and return
-    rescue Exception => e
-      logger.error e.message + "\n" + e.backtrace.join("\n")
-      save_yaml_error("Undefined yaml error") and return
+    job_types = config_processor.types
+    builds_without_retry.sort_by do |build|
+      [job_types.index(build.job_type) || -1, build.name || ""]
     end
+  end
 
-    deploy_builds_for_ref.each do |build_attrs|
-      builds.create!({
-        project: project,
-        name: build_attrs[:name],
-        commands: build_attrs[:script],
-        tag_list: build_attrs[:tags],
-        options: build_attrs[:options],
-        allow_failure: build_attrs[:allow_failure],
-        deploy: true
-      })
-    end
+  def retried_builds
+    @retried_builds ||= (builds.order(id: :desc) - builds_without_retry)
   end
 
   def status
@@ -225,6 +229,13 @@ class Commit < ActiveRecord::Base
 
   def config_processor
     @config_processor ||= GitlabCiYamlProcessor.new(push_data[:ci_yaml_file] || project.generated_yaml_config)
+  rescue GitlabCiYamlProcessor::ValidationError => e
+    save_yaml_error(e.message)
+    nil
+  rescue Exception => e
+    logger.error e.message + "\n" + e.backtrace.join("\n")
+    save_yaml_error("Undefined yaml error")
+    nil
   end
 
   def skip_ci?
@@ -235,6 +246,7 @@ class Commit < ActiveRecord::Base
   private
 
   def save_yaml_error(error)
+    return unless self.yaml_errors?
     self.yaml_errors = error
     save
   end