Merge branch 'ci-and-ce-sitting-in-a-tree-k-i-s-s-i-n-g' into 'master'

Merge CI into CE

First step of #2164.

- [x] Merge latest CE master
- [x] Make application start
- [x] Re-use gitlab sessions (remove CI oauth part)
- [x] Get rid of gitlab_ci.yml config
- [x] Make tests start
- [x] Make most CI features works
- [x] Make tests green
- [x] Write migration documentation
- [x] Add CI builds to CE backup

See merge request !1204

21 files changed, 1376 insertions, 0 deletions

diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
new file mode 100644
index 00000000000..ac6d667cf8d
--- /dev/null
+++ b/lib/ci/ansi2html.rb
@@ -0,0 +1,224 @@
+# ANSI color library
+#
+# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code
+module Ci
+  module Ansi2html
+    # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107)
+    COLOR = {
+      0 => 'black', # not that this is gray in the intense color table
+      1 => 'red',
+      2 => 'green',
+      3 => 'yellow',
+      4 => 'blue',
+      5 => 'magenta',
+      6 => 'cyan',
+      7 => 'white', # not that this is gray in the dark (aka default) color table
+    }
+
+    STYLE_SWITCHES = {
+      bold:       0x01,
+      italic:     0x02,
+      underline:  0x04,
+      conceal:    0x08,
+      cross:      0x10,
+    }
+
+    def self.convert(ansi)
+      Converter.new().convert(ansi)
+    end
+
+    class Converter
+      def on_0(s) reset()                            end
+      def on_1(s) enable(STYLE_SWITCHES[:bold])      end
+      def on_3(s) enable(STYLE_SWITCHES[:italic])    end
+      def on_4(s) enable(STYLE_SWITCHES[:underline]) end
+      def on_8(s) enable(STYLE_SWITCHES[:conceal])   end
+      def on_9(s) enable(STYLE_SWITCHES[:cross])     end
+
+      def on_21(s) disable(STYLE_SWITCHES[:bold])      end
+      def on_22(s) disable(STYLE_SWITCHES[:bold])      end
+      def on_23(s) disable(STYLE_SWITCHES[:italic])    end
+      def on_24(s) disable(STYLE_SWITCHES[:underline]) end
+      def on_28(s) disable(STYLE_SWITCHES[:conceal])   end
+      def on_29(s) disable(STYLE_SWITCHES[:cross])     end
+
+      def on_30(s) set_fg_color(0) end
+      def on_31(s) set_fg_color(1) end
+      def on_32(s) set_fg_color(2) end
+      def on_33(s) set_fg_color(3) end
+      def on_34(s) set_fg_color(4) end
+      def on_35(s) set_fg_color(5) end
+      def on_36(s) set_fg_color(6) end
+      def on_37(s) set_fg_color(7) end
+      def on_38(s) set_fg_color_256(s) end
+      def on_39(s) set_fg_color(9) end
+
+      def on_40(s) set_bg_color(0) end
+      def on_41(s) set_bg_color(1) end
+      def on_42(s) set_bg_color(2) end
+      def on_43(s) set_bg_color(3) end
+      def on_44(s) set_bg_color(4) end
+      def on_45(s) set_bg_color(5) end
+      def on_46(s) set_bg_color(6) end
+      def on_47(s) set_bg_color(7) end
+      def on_48(s) set_bg_color_256(s) end
+      def on_49(s) set_bg_color(9) end
+
+      def on_90(s) set_fg_color(0, 'l') end
+      def on_91(s) set_fg_color(1, 'l') end
+      def on_92(s) set_fg_color(2, 'l') end
+      def on_93(s) set_fg_color(3, 'l') end
+      def on_94(s) set_fg_color(4, 'l') end
+      def on_95(s) set_fg_color(5, 'l') end
+      def on_96(s) set_fg_color(6, 'l') end
+      def on_97(s) set_fg_color(7, 'l') end
+      def on_99(s) set_fg_color(9, 'l') end
+
+      def on_100(s) set_bg_color(0, 'l') end
+      def on_101(s) set_bg_color(1, 'l') end
+      def on_102(s) set_bg_color(2, 'l') end
+      def on_103(s) set_bg_color(3, 'l') end
+      def on_104(s) set_bg_color(4, 'l') end
+      def on_105(s) set_bg_color(5, 'l') end
+      def on_106(s) set_bg_color(6, 'l') end
+      def on_107(s) set_bg_color(7, 'l') end
+      def on_109(s) set_bg_color(9, 'l') end
+
+      def convert(ansi)
+        @out = ""
+        @n_open_tags = 0
+        reset()
+
+        s = StringScanner.new(ansi.gsub("<", "&lt;"))
+        while(!s.eos?)
+          if s.scan(/\e([@-_])(.*?)([@-~])/)
+            handle_sequence(s)
+          else
+            @out << s.scan(/./m)
+          end
+        end
+
+        close_open_tags()
+        @out
+      end
+
+      def handle_sequence(s)
+        indicator = s[1]
+        commands = s[2].split ';'
+        terminator = s[3]
+
+        # We are only interested in color and text style changes - triggered by
+        # sequences starting with '\e[' and ending with 'm'. Any other control
+        # sequence gets stripped (including stuff like "delete last line")
+        return unless indicator == '[' and terminator == 'm'
+
+        close_open_tags()
+
+        if commands.empty?()
+          reset()
+          return
+        end
+
+        evaluate_command_stack(commands)
+
+        css_classes = []
+
+        unless @fg_color.nil?
+          fg_color = @fg_color
+          # Most terminals show bold colored text in the light color variant
+          # Let's mimic that here
+          if @style_mask & STYLE_SWITCHES[:bold] != 0
+            fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1')
+          end
+          css_classes << fg_color
+        end
+        css_classes << @bg_color unless @bg_color.nil?
+
+        STYLE_SWITCHES.each do |css_class, flag|
+          css_classes << "term-#{css_class}" if @style_mask & flag != 0
+        end
+
+        open_new_tag(css_classes) if css_classes.length > 0
+      end
+
+      def evaluate_command_stack(stack)
+        return unless command = stack.shift()
+
+        if self.respond_to?("on_#{command}", true)
+          self.send("on_#{command}", stack)
+        end
+
+        evaluate_command_stack(stack)
+      end
+
+      def open_new_tag(css_classes)
+        @out << %{<span class="#{css_classes.join(' ')}">}
+        @n_open_tags += 1
+      end
+
+      def close_open_tags
+        while @n_open_tags > 0
+          @out << %{</span>}
+          @n_open_tags -= 1
+        end
+      end
+
+      def reset
+        @fg_color = nil
+        @bg_color = nil
+        @style_mask = 0
+      end
+
+      def enable(flag)
+        @style_mask |= flag
+      end
+
+      def disable(flag)
+        @style_mask &= ~flag
+      end
+
+      def set_fg_color(color_index, prefix = nil)
+        @fg_color = get_term_color_class(color_index, ["fg", prefix])
+      end
+
+      def set_bg_color(color_index, prefix = nil)
+        @bg_color = get_term_color_class(color_index, ["bg", prefix])
+      end
+
+      def get_term_color_class(color_index, prefix)
+        color_name = COLOR[color_index]
+        return nil if color_name.nil?
+
+        get_color_class(["term", prefix, color_name])
+      end
+
+      def set_fg_color_256(command_stack)
+        css_class = get_xterm_color_class(command_stack, "fg")
+        @fg_color = css_class unless css_class.nil?
+      end
+
+      def set_bg_color_256(command_stack)
+        css_class = get_xterm_color_class(command_stack, "bg")
+        @bg_color = css_class unless css_class.nil?
+      end
+
+      def get_xterm_color_class(command_stack, prefix)
+        # the 38 and 48 commands have to be followed by "5" and the color index
+        return unless command_stack.length >= 2
+        return unless command_stack[0] == "5"
+
+        command_stack.shift() # ignore the "5" command
+        color_index = command_stack.shift().to_i
+
+        return unless color_index >= 0
+        return unless color_index <= 255
+
+        get_color_class(["xterm", prefix, color_index])
+      end
+
+      def get_color_class(segments)
+        [segments].flatten.compact.join('-')
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
new file mode 100644
index 00000000000..172c6f22164
--- /dev/null
+++ b/lib/ci/api/api.rb
@@ -0,0 +1,39 @@
+Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file}
+
+module Ci
+  module API
+    class API < Grape::API
+      include APIGuard
+      version 'v1', using: :path
+
+      rescue_from ActiveRecord::RecordNotFound do
+        rack_response({ 'message' => '404 Not found' }.to_json, 404)
+      end
+
+      rescue_from :all do |exception|
+        # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
+        # why is this not wrapped in something reusable?
+        trace = exception.backtrace
+
+        message = "\n#{exception.class} (#{exception.message}):\n"
+        message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code)
+        message << "  " << trace.join("\n  ")
+
+        API.logger.add Logger::FATAL, message
+        rack_response({ 'message' => '500 Internal Server Error' }, 500)
+      end
+
+      format :json
+
+      helpers Helpers
+      helpers ::API::APIHelpers
+
+      mount Builds
+      mount Commits
+      mount Runners
+      mount Projects
+      mount Forks
+      mount Triggers
+    end
+  end
+end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
new file mode 100644
index 00000000000..83ca1e6481c
--- /dev/null
+++ b/lib/ci/api/builds.rb
@@ -0,0 +1,53 @@
+module Ci
+  module API
+    # Builds API
+    class Builds < Grape::API
+      resource :builds do
+        # Runs oldest pending build by runner - Runners only
+        #
+        # Parameters:
+        #   token (required) - The uniq token of runner
+        #
+        # Example Request:
+        #   POST /builds/register
+        post "register" do
+          authenticate_runner!
+          update_runner_last_contact
+          required_attributes! [:token]
+          not_found! unless current_runner.active?
+
+          build = Ci::RegisterBuildService.new.execute(current_runner)
+
+          if build
+            update_runner_info
+            present build, with: Entities::Build
+          else
+            not_found!
+          end
+        end
+
+        # Update an existing build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a project
+        #   state (optional) - The state of a build
+        #   trace (optional) - The trace of a build
+        # Example Request:
+        #   PUT /builds/:id
+        put ":id" do
+          authenticate_runner!
+          update_runner_last_contact
+          build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id])
+          build.update_attributes(trace: params[:trace]) if params[:trace]
+
+          case params[:state].to_s
+          when 'success'
+            build.success
+          when 'failed'
+            build.drop
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/commits.rb b/lib/ci/api/commits.rb
new file mode 100644
index 00000000000..bac463a5909
--- /dev/null
+++ b/lib/ci/api/commits.rb
@@ -0,0 +1,66 @@
+module Ci
+  module API
+    class Commits < Grape::API
+      resource :commits do
+        # Get list of commits per project
+        #
+        # Parameters:
+        #   project_id (required) - The ID of a project
+        #   project_token (requires) - Project token
+        #   page (optional)
+        #   per_page (optional) - items per request (default is 20)
+        #
+        get do
+          required_attributes! [:project_id, :project_token]
+          project = Ci::Project.find(params[:project_id])
+          authenticate_project_token!(project)
+
+          commits = project.commits.page(params[:page]).per(params[:per_page] || 20)
+          present commits, with: Entities::CommitWithBuilds
+        end
+
+        # Create a commit
+        #
+        # Parameters:
+        #   project_id (required) - The ID of a project
+        #   project_token (requires) - Project token
+        #   data (required) - GitLab push data
+        #
+        #   Sample GitLab push data:
+        #   {
+        #     "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
+        #     "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+        #     "ref": "refs/heads/master",
+        #     "commits": [
+        #       {
+        #         "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
+        #         "message": "Update Catalan translation to e38cb41.",
+        #         "timestamp": "2011-12-12T14:27:31+02:00",
+        #         "url": "http://localhost/diaspora/commits/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
+        #         "author": {
+        #           "name": "Jordi Mallach",
+        #           "email": "jordi@softcatala.org",
+        #         }
+        #       }, .... more commits
+        #     ]
+        #   }
+        #
+        # Example Request:
+        #   POST /commits
+        post do
+          required_attributes! [:project_id, :data, :project_token]
+          project = Ci::Project.find(params[:project_id])
+          authenticate_project_token!(project)
+          commit = Ci::CreateCommitService.new.execute(project, params[:data])
+
+          if commit.persisted?
+            present commit, with: Entities::CommitWithBuilds
+          else
+            errors = commit.errors.full_messages.join(", ")
+            render_api_error!(errors, 400)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
new file mode 100644
index 00000000000..f47bc1236b8
--- /dev/null
+++ b/lib/ci/api/entities.rb
@@ -0,0 +1,56 @@
+module Ci
+  module API
+    module Entities
+      class Commit < Grape::Entity
+        expose :id, :ref, :sha, :project_id, :before_sha, :created_at
+        expose :status, :finished_at, :duration
+        expose :git_commit_message, :git_author_name, :git_author_email
+      end
+
+      class CommitWithBuilds < Commit
+        expose :builds
+      end
+
+      class Build < Grape::Entity
+        expose :id, :commands, :ref, :sha, :project_id, :repo_url,
+          :before_sha, :allow_git_fetch, :project_name
+
+        expose :options do |model|
+          model.options
+        end
+
+        expose :timeout do |model|
+          model.timeout
+        end
+
+        expose :variables
+      end
+
+      class Runner < Grape::Entity
+        expose :id, :token
+      end
+
+      class Project < Grape::Entity
+        expose :id, :name, :token, :default_ref, :gitlab_url, :path,
+          :always_build, :polling_interval, :public, :ssh_url_to_repo, :gitlab_id
+
+        expose :timeout do |model|
+          model.timeout
+        end
+      end
+
+      class RunnerProject < Grape::Entity
+        expose :id, :project_id, :runner_id
+      end
+
+      class WebHook < Grape::Entity
+        expose :id, :project_id, :url
+      end
+
+      class TriggerRequest < Grape::Entity
+        expose :id, :variables
+        expose :commit, using: Commit
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/forks.rb b/lib/ci/api/forks.rb
new file mode 100644
index 00000000000..152883a599f
--- /dev/null
+++ b/lib/ci/api/forks.rb
@@ -0,0 +1,37 @@
+module Ci
+  module API
+    class Forks < Grape::API
+      resource :forks do
+        # Create a fork
+        #
+        # Parameters:
+        #   project_id (required) - The ID of a project
+        #   project_token (requires) - Project token
+        #   private_token(required) - User private token
+        #   data (required) - GitLab project data (name_with_namespace, web_url, default_branch, ssh_url_to_repo)
+        #
+        #
+        # Example Request:
+        #   POST /forks
+        post do
+          required_attributes! [:project_id, :data, :project_token, :private_token]
+          project = Ci::Project.find_by!(gitlab_id: params[:project_id])
+          authenticate_project_token!(project)
+
+          fork = Ci::CreateProjectService.new.execute(
+            current_user,
+            params[:data],
+            Ci::RoutesHelper.ci_project_url(":project_id"),
+            project
+          )
+
+          if fork
+            present fork, with: Entities::Project
+          else
+            not_found!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
new file mode 100644
index 00000000000..9197f917d73
--- /dev/null
+++ b/lib/ci/api/helpers.rb
@@ -0,0 +1,33 @@
+module Ci
+  module API
+    module Helpers
+      def authenticate_runners!
+        forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN
+      end
+
+      def authenticate_runner!
+        forbidden! unless current_runner
+      end
+
+      def authenticate_project_token!(project)
+        forbidden! unless project.valid_token?(params[:project_token])
+      end
+
+      def update_runner_last_contact
+        if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY
+          current_runner.update_attributes(contacted_at: Time.now)
+        end
+      end
+
+      def current_runner
+        @runner ||= Runner.find_by_token(params[:token].to_s)
+      end
+
+      def update_runner_info
+        return unless params["info"].present?
+        info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
+        current_runner.update(info)
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb
new file mode 100644
index 00000000000..66bcf65e8c4
--- /dev/null
+++ b/lib/ci/api/projects.rb
@@ -0,0 +1,210 @@
+module Ci
+  module API
+    # Projects API
+    class Projects < Grape::API
+      before { authenticate! }
+
+      resource :projects do
+        # Register new webhook for project
+        #
+        # Parameters
+        #   project_id (required) - The ID of a project
+        #   web_hook (required) - WebHook URL
+        # Example Request
+        #   POST /projects/:project_id/webhooks
+        post ":project_id/webhooks" do
+          required_attributes! [:web_hook]
+
+          project = Ci::Project.find(params[:project_id])
+
+          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
+
+          web_hook = project.web_hooks.new({ url: params[:web_hook] })
+
+          if web_hook.save
+            present web_hook, with: Entities::WebHook
+          else
+            errors = web_hook.errors.full_messages.join(", ")
+            render_api_error!(errors, 400)
+          end
+        end
+
+        # Retrieve all Gitlab CI projects that the user has access to
+        #
+        # Example Request:
+        #   GET /projects
+        get do
+          gitlab_projects = current_user.authorized_projects
+          gitlab_projects = filter_projects(gitlab_projects)
+          gitlab_projects = paginate gitlab_projects
+
+          ids = gitlab_projects.map { |project| project.id }
+
+          projects = Ci::Project.where("gitlab_id IN (?)", ids).load
+          present projects, with: Entities::Project
+        end
+
+        # Retrieve all Gitlab CI projects that the user owns
+        #
+        # Example Request:
+        #   GET /projects/owned
+        get "owned" do
+          gitlab_projects = current_user.owned_projects
+          gitlab_projects = filter_projects(gitlab_projects)
+          gitlab_projects = paginate gitlab_projects
+
+          ids = gitlab_projects.map { |project| project.id }
+
+          projects = Ci::Project.where("gitlab_id IN (?)", ids).load
+          present projects, with: Entities::Project
+        end
+
+        # Retrieve info for a Gitlab CI project
+        #
+        # Parameters:
+        #   id (required) - The ID of a project
+        # Example Request:
+        #   GET /projects/:id
+        get ":id" do
+          project = Ci::Project.find(params[:id])
+          unauthorized! unless can?(current_user, :read_project, project.gl_project)
+
+          present project, with: Entities::Project
+        end
+
+        # Create Gitlab CI project using Gitlab project info
+        #
+        # Parameters:
+        #   name (required)            - The name of the project
+        #   gitlab_id (required)       - The gitlab id of the project
+        #   path (required)            - The gitlab project path, ex. randx/six
+        #   ssh_url_to_repo (required) - The gitlab ssh url to the repo
+        #   default_ref                - The branch to run against (defaults to `master`)
+        # Example Request:
+        #   POST /projects
+        post do
+          required_attributes! [:name, :gitlab_id, :ssh_url_to_repo]
+
+          filtered_params = {
+            name:            params[:name],
+            gitlab_id:       params[:gitlab_id],
+            # we accept gitlab_url for backward compatibility for a while (added to 7.11)
+            path:            params[:path] || params[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1'),
+            default_ref:     params[:default_ref] || 'master',
+            ssh_url_to_repo: params[:ssh_url_to_repo]
+          }
+
+          project = Ci::Project.new(filtered_params)
+          project.build_missing_services
+
+          if project.save
+            present project, with: Entities::Project
+          else
+            errors = project.errors.full_messages.join(", ")
+            render_api_error!(errors, 400)
+          end
+        end
+
+        # Update a Gitlab CI project
+        #
+        # Parameters:
+        #   id (required)   - The ID of a project
+        #   name            - The name of the project
+        #   gitlab_id       - The gitlab id of the project
+        #   path            - The gitlab project path, ex. randx/six
+        #   ssh_url_to_repo - The gitlab ssh url to the repo
+        #   default_ref     - The branch to run against (defaults to `master`)
+        # Example Request:
+        #   PUT /projects/:id
+        put ":id" do
+          project = Ci::Project.find(params[:id])
+
+          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
+
+          attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo]
+
+          # we accept gitlab_url for backward compatibility for a while (added to 7.11)
+          if attrs[:gitlab_url] && !attrs[:path]
+            attrs[:path] = attrs[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1')
+          end
+
+          if project.update_attributes(attrs)
+            present project, with: Entities::Project
+          else
+            errors = project.errors.full_messages.join(", ")
+            render_api_error!(errors, 400)
+          end
+        end
+
+        # Remove a Gitlab CI project
+        #
+        # Parameters:
+        #   id (required) - The ID of a project
+        # Example Request:
+        #   DELETE /projects/:id
+        delete ":id" do
+          project = Ci::Project.find(params[:id])
+
+          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
+
+          project.destroy
+        end
+
+        # Link a Gitlab CI project to a runner
+        #
+        # Parameters:
+        #   id (required) - The ID of a CI project
+        #   runner_id (required) - The ID of a runner
+        # Example Request:
+        #   POST /projects/:id/runners/:runner_id
+        post ":id/runners/:runner_id" do
+          project = Ci::Project.find(params[:id])
+          runner  = Ci::Runner.find(params[:runner_id])
+
+          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
+
+          options = {
+            project_id: project.id,
+            runner_id:  runner.id
+          }
+
+          runner_project = Ci::RunnerProject.new(options)
+
+          if runner_project.save
+            present runner_project, with: Entities::RunnerProject
+          else
+            errors = project.errors.full_messages.join(", ")
+            render_api_error!(errors, 400)
+          end
+        end
+
+        # Remove a Gitlab CI project from a runner
+        #
+        # Parameters:
+        #   id (required) - The ID of a CI project
+        #   runner_id (required) - The ID of a runner
+        # Example Request:
+        #   DELETE /projects/:id/runners/:runner_id
+        delete ":id/runners/:runner_id" do
+          project = Ci::Project.find(params[:id])
+          runner  = Ci::Runner.find(params[:runner_id])
+
+          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
+
+          options = {
+            project_id: project.id,
+            runner_id:  runner.id
+          }
+
+          runner_project = Ci::RunnerProject.find_by(options)
+
+          if runner_project.present?
+            runner_project.destroy
+          else
+            not_found!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb
new file mode 100644
index 00000000000..1466fe4356e
--- /dev/null
+++ b/lib/ci/api/runners.rb
@@ -0,0 +1,69 @@
+module Ci
+  module API
+    # Runners API
+    class Runners < Grape::API
+      resource :runners do
+        # Get list of all available runners
+        #
+        # Example Request:
+        #   GET /runners
+        get do
+          authenticate!
+          runners = Ci::Runner.all
+
+          present runners, with: Entities::Runner
+        end
+
+        # Delete runner
+        # Parameters:
+        #   token (required) - The unique token of runner
+        #
+        # Example Request:
+        #   GET /runners/delete
+        delete "delete" do
+          required_attributes! [:token]
+          authenticate_runner!
+          Ci::Runner.find_by_token(params[:token]).destroy
+        end
+
+        # Register a new runner
+        #
+        # Note: This is an "internal" API called when setting up
+        # runners, so it is authenticated differently.
+        #
+        # Parameters:
+        #   token (required) - The unique token of runner
+        #
+        # Example Request:
+        #   POST /runners/register
+        post "register" do
+          required_attributes! [:token]
+
+          runner =
+            if params[:token] == GitlabCi::REGISTRATION_TOKEN
+              # Create shared runner. Requires admin access
+              Ci::Runner.create(
+                description: params[:description],
+                tag_list: params[:tag_list],
+                is_shared: true
+              )
+            elsif project = Ci::Project.find_by(token: params[:token])
+              # Create a specific runner for project.
+              project.runners.create(
+                description: params[:description],
+                tag_list: params[:tag_list]
+              )
+            end
+
+          return forbidden! unless runner
+
+          if runner.id
+            present runner, with: Entities::Runner
+          else
+            not_found!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb
new file mode 100644
index 00000000000..40907d6db54
--- /dev/null
+++ b/lib/ci/api/triggers.rb
@@ -0,0 +1,49 @@
+module Ci
+  module API
+    # Build Trigger API
+    class Triggers < Grape::API
+      resource :projects do
+        # Trigger a GitLab CI project build
+        #
+        # Parameters:
+        #   id (required) - The ID of a CI project
+        #   ref (required) - The name of project's branch or tag
+        #   token (required) - The uniq token of trigger
+        # Example Request:
+        #   POST /projects/:id/ref/:ref/trigger
+        post ":id/refs/:ref/trigger" do
+          required_attributes! [:token]
+
+          project = Ci::Project.find(params[:id])
+          trigger = Ci::Trigger.find_by_token(params[:token].to_s)
+          not_found! unless project && trigger
+          unauthorized! unless trigger.project == project
+
+          # validate variables
+          variables = params[:variables]
+          if variables
+            unless variables.is_a?(Hash)
+              render_api_error!('variables needs to be a hash', 400)
+            end
+
+            unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+              render_api_error!('variables needs to be a map of key-valued strings', 400)
+            end
+
+            # convert variables from Mash to Hash
+            variables = variables.to_h
+          end
+
+          # create request and trigger builds
+          trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables)
+          if trigger_request
+            present trigger_request, with: Entities::TriggerRequest
+          else
+            errors = 'No builds created'
+            render_api_error!(errors, 400)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/assets/.gitkeep b/lib/ci/assets/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/lib/ci/assets/.gitkeep
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
new file mode 100644
index 00000000000..915a4f526a6
--- /dev/null
+++ b/lib/ci/charts.rb
@@ -0,0 +1,71 @@
+module Ci
+  module Charts
+    class Chart
+      attr_reader :labels, :total, :success, :project, :build_times
+
+      def initialize(project)
+        @labels = []
+        @total = []
+        @success = []
+        @build_times = []
+        @project = project
+
+        collect
+      end
+
+
+      def push(from, to, format)
+        @labels << from.strftime(format)
+        @total << project.builds.
+          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from).
+          count(:all)
+        @success << project.builds.
+          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from).
+          success.count(:all)
+      end
+    end
+
+    class YearChart < Chart
+      def collect
+        13.times do |i|
+          start_month = (Date.today.years_ago(1) + i.month).beginning_of_month
+          end_month = start_month.end_of_month
+
+          push(start_month, end_month, "%d %B %Y")
+        end
+      end
+    end
+
+    class MonthChart < Chart
+      def collect
+        30.times do |i|
+          start_day = Date.today - 30.days + i.days
+          end_day = Date.today - 30.days + i.day + 1.day
+
+          push(start_day, end_day, "%d %B")
+        end
+      end
+    end
+
+    class WeekChart < Chart
+      def collect
+        7.times do |i|
+          start_day = Date.today - 7.days + i.days
+          end_day = Date.today - 7.days + i.day + 1.day
+
+          push(start_day, end_day, "%d %B")
+        end
+      end
+    end
+
+    class BuildTime < Chart
+      def collect
+        commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30)
+        commits.each do |commit|
+          @labels << commit.short_sha
+          @build_times << (commit.duration / 60)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/current_settings.rb b/lib/ci/current_settings.rb
new file mode 100644
index 00000000000..fd78b024970
--- /dev/null
+++ b/lib/ci/current_settings.rb
@@ -0,0 +1,22 @@
+module Ci
+  module CurrentSettings
+    def current_application_settings
+      key = :ci_current_application_settings
+
+      RequestStore.store[key] ||= begin
+        if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('ci_application_settings')
+          Ci::ApplicationSetting.current || Ci::ApplicationSetting.create_from_defaults
+        else
+          fake_application_settings
+        end
+      end
+    end
+
+    def fake_application_settings
+      OpenStruct.new(
+        all_broken_builds: Ci::Settings.gitlab_ci['all_broken_builds'],
+        add_pusher: Ci::Settings.gitlab_ci['add_pusher'],
+      )
+    end
+  end
+end
diff --git a/lib/ci/git.rb b/lib/ci/git.rb
new file mode 100644
index 00000000000..7acc3f38edb
--- /dev/null
+++ b/lib/ci/git.rb
@@ -0,0 +1,5 @@
+module Ci
+  module Git
+    BLANK_SHA = '0' * 40
+  end
+end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
new file mode 100644
index 00000000000..e625e790df8
--- /dev/null
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -0,0 +1,198 @@
+module Ci
+  class GitlabCiYamlProcessor
+    class ValidationError < StandardError;end
+
+    DEFAULT_STAGES = %w(build test deploy)
+    DEFAULT_STAGE = 'test'
+    ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage]
+
+    attr_reader :before_script, :image, :services, :variables
+
+    def initialize(config)
+      @config = YAML.load(config)
+
+      unless @config.is_a? Hash
+        raise ValidationError, "YAML should be a hash"
+      end
+
+      @config = @config.deep_symbolize_keys
+
+      initial_parsing
+
+      validate!
+    end
+
+    def builds_for_stage_and_ref(stage, ref, tag = false)
+      builds.select{|build| build[:stage] == stage && process?(build[:only], build[:except], ref, tag)}
+    end
+
+    def builds
+      @jobs.map do |name, job|
+        build_job(name, job)
+      end
+    end
+
+    def stages
+      @stages || DEFAULT_STAGES
+    end
+
+    private
+
+    def initial_parsing
+      @before_script = @config[:before_script] || []
+      @image = @config[:image]
+      @services = @config[:services]
+      @stages = @config[:stages] || @config[:types]
+      @variables = @config[:variables] || {}
+      @config.except!(*ALLOWED_YAML_KEYS)
+
+      # anything that doesn't have script is considered as unknown
+      @config.each do |name, param|
+        raise ValidationError, "Unknown parameter: #{name}" unless param.is_a?(Hash) && param.has_key?(:script)
+      end
+
+      unless @config.values.any?{|job| job.is_a?(Hash)}
+        raise ValidationError, "Please define at least one job"
+      end
+
+      @jobs = {}
+      @config.each do |key, job|
+        stage = job[:stage] || job[:type] || DEFAULT_STAGE
+        @jobs[key] = { stage: stage }.merge(job)
+      end
+    end
+
+    def process?(only_params, except_params, ref, tag)
+      return true if only_params.nil? && except_params.nil?
+
+      if only_params
+        return true if tag && only_params.include?("tags")
+        return true if !tag && only_params.include?("branches")
+        
+        only_params.find do |pattern|
+          match_ref?(pattern, ref)
+        end
+      else
+        return false if tag && except_params.include?("tags")
+        return false if !tag && except_params.include?("branches")
+
+        except_params.each do |pattern|
+          return false if match_ref?(pattern, ref)
+        end
+      end
+    end
+
+    def build_job(name, job)
+      {
+        stage: job[:stage],
+        script: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}",
+        tags: job[:tags] || [],
+        name: name,
+        only: job[:only],
+        except: job[:except],
+        allow_failure: job[:allow_failure] || false,
+        options: {
+          image: job[:image] || @image,
+          services: job[:services] || @services
+        }.compact
+      }
+    end
+
+    def match_ref?(pattern, ref)
+      if pattern.first == "/" && pattern.last == "/"
+        Regexp.new(pattern[1...-1]) =~ ref
+      else
+        pattern == ref
+      end
+    end
+
+    def normalize_script(script)
+      if script.is_a? Array
+        script.join("\n")
+      else
+        script
+      end
+    end
+
+    def validate!
+      unless validate_array_of_strings(@before_script)
+        raise ValidationError, "before_script should be an array of strings"
+      end
+
+      unless @image.nil? || @image.is_a?(String)
+        raise ValidationError, "image should be a string"
+      end
+
+      unless @services.nil? || validate_array_of_strings(@services)
+        raise ValidationError, "services should be an array of strings"
+      end
+
+      unless @stages.nil? || validate_array_of_strings(@stages)
+        raise ValidationError, "stages should be an array of strings"
+      end
+
+      unless @variables.nil? || validate_variables(@variables)
+        raise ValidationError, "variables should be a map of key-valued strings"
+      end
+
+      @jobs.each do |name, job|
+        validate_job!("#{name} job", job)
+      end
+
+      true
+    end
+
+    def validate_job!(name, job)
+      job.keys.each do |key|
+        unless ALLOWED_JOB_KEYS.include? key
+          raise ValidationError, "#{name}: unknown parameter #{key}"
+        end
+      end
+
+      if !job[:script].is_a?(String) && !validate_array_of_strings(job[:script])
+        raise ValidationError, "#{name}: script should be a string or an array of a strings"
+      end
+
+      if job[:stage]
+        unless job[:stage].is_a?(String) && job[:stage].in?(stages)
+          raise ValidationError, "#{name}: stage parameter should be #{stages.join(", ")}"
+        end
+      end
+
+      if job[:image] && !job[:image].is_a?(String)
+        raise ValidationError, "#{name}: image should be a string"
+      end
+
+      if job[:services] && !validate_array_of_strings(job[:services])
+        raise ValidationError, "#{name}: services should be an array of strings"
+      end
+
+      if job[:tags] && !validate_array_of_strings(job[:tags])
+        raise ValidationError, "#{name}: tags parameter should be an array of strings"
+      end
+
+      if job[:only] && !validate_array_of_strings(job[:only])
+        raise ValidationError, "#{name}: only parameter should be an array of strings"
+      end
+
+      if job[:except] && !validate_array_of_strings(job[:except])
+        raise ValidationError, "#{name}: except parameter should be an array of strings"
+      end
+
+      if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+        raise ValidationError, "#{name}: allow_failure parameter should be an boolean"
+      end
+    end
+
+    private
+
+    def validate_array_of_strings(values)
+      values.is_a?(Array) && values.all? {|tag| tag.is_a?(String)}
+    end
+
+    def validate_variables(variables)
+      variables.is_a?(Hash) && variables.all? {|key, value| key.is_a?(Symbol) && value.is_a?(String)}
+    end
+  end
+end
diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb
new file mode 100644
index 00000000000..74f592dcaea
--- /dev/null
+++ b/lib/ci/migrate/database.rb
@@ -0,0 +1,67 @@
+require 'yaml'
+
+module Ci
+  module Migrate
+    class Database
+      attr_reader :config
+
+      def initialize
+        @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
+      end
+
+      def restore(ci_dump)
+        puts 'Deleting all CI related data ... '
+        truncate_ci_tables
+
+        puts 'Restoring CI data ... '
+        case config["adapter"]
+          when /^mysql/ then
+            print "Restoring MySQL database #{config['database']} ... "
+            # Workaround warnings from MySQL 5.6 about passwords on cmd line
+            ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
+            system('mysql', *mysql_args, config['database'], in: ci_dump)
+          when "postgresql" then
+            puts "Restoring PostgreSQL database #{config['database']} ... "
+            pg_env
+            system('psql', config['database'], '-f', ci_dump)
+        end
+      end
+
+      protected
+
+      def truncate_ci_tables
+        c = ActiveRecord::Base.connection
+        c.tables.select { |t| t.start_with?('ci_') }.each do |table|
+          puts "Deleting data from #{table}..."
+          c.execute("DELETE FROM #{table}")
+        end
+      end
+
+      def mysql_args
+        args = {
+          'host' => '--host',
+          'port' => '--port',
+          'socket' => '--socket',
+          'username' => '--user',
+          'encoding' => '--default-character-set'
+        }
+        args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
+      end
+
+      def pg_env
+        ENV['PGUSER'] = config["username"] if config["username"]
+        ENV['PGHOST'] = config["host"] if config["host"]
+        ENV['PGPORT'] = config["port"].to_s if config["port"]
+        ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
+      end
+
+      def report_success(success)
+        if success
+          puts '[DONE]'.green
+        else
+          puts '[FAILED]'.red
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb
new file mode 100644
index 00000000000..f4114c698d2
--- /dev/null
+++ b/lib/ci/migrate/tags.rb
@@ -0,0 +1,49 @@
+require 'yaml'
+
+module Ci
+  module Migrate
+    class Tags
+      def restore
+        puts 'Migrating tags for Runners... '
+        list_objects('Runner').each do |id|
+          putc '.'
+          runner = Ci::Runner.find_by_id(id)
+          if runner
+            tags = list_tags('Runner', id)
+            runner.update_attributes(tag_list: tags)
+          end
+        end
+        puts ''
+
+        puts 'Migrating tags for Builds... '
+        list_objects('Build').each do |id|
+          putc '.'
+          build = Ci::Build.find_by_id(id)
+          if build
+            tags = list_tags('Build', id)
+            build.update_attributes(tag_list: tags)
+          end
+        end
+        puts ''
+      end
+
+      protected
+
+      def list_objects(type)
+        ids = ActiveRecord::Base.connection.select_all(
+          "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}"
+        )
+        ids.map { |id| id['taggable_id'] }
+      end
+
+      def list_tags(type, id)
+        tags = ActiveRecord::Base.connection.select_all(
+          'select ci_tags.name from ci_tags ' +
+            'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
+            "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = \"tags\""
+        )
+        tags.map { |tag| tag['name'] }
+      end
+    end
+  end
+end
diff --git a/lib/ci/model.rb b/lib/ci/model.rb
new file mode 100644
index 00000000000..c42a0ad36db
--- /dev/null
+++ b/lib/ci/model.rb
@@ -0,0 +1,11 @@
+module Ci
+  module Model
+    def table_name_prefix
+      "ci_"
+    end
+
+    def model_name
+      @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last)
+    end
+  end
+end
diff --git a/lib/ci/scheduler.rb b/lib/ci/scheduler.rb
new file mode 100644
index 00000000000..ee0958f4be1
--- /dev/null
+++ b/lib/ci/scheduler.rb
@@ -0,0 +1,16 @@
+module Ci
+  class Scheduler
+    def perform
+      projects = Ci::Project.where(always_build: true).all
+      projects.each do |project|
+        last_commit = project.commits.last
+        next unless last_commit && last_commit.last_build
+
+        interval = project.polling_interval
+        if (last_commit.last_build.created_at + interval.hours) < Time.now
+          last_commit.retry
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/static_model.rb b/lib/ci/static_model.rb
new file mode 100644
index 00000000000..bb2bdbed495
--- /dev/null
+++ b/lib/ci/static_model.rb
@@ -0,0 +1,49 @@
+# Provides an ActiveRecord-like interface to a model whose data is not persisted to a database.
+module Ci
+  module StaticModel
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      # Used by ActiveRecord's polymorphic association to set object_id
+      def primary_key
+        'id'
+      end
+
+      # Used by ActiveRecord's polymorphic association to set object_type
+      def base_class
+        self
+      end
+    end
+
+    # Used by AR for fetching attributes
+    #
+    # Pass it along if we respond to it.
+    def [](key)
+      send(key) if respond_to?(key)
+    end
+
+    def to_param
+      id
+    end
+
+    def new_record?
+      false
+    end
+
+    def persisted?
+      false
+    end
+
+    def destroyed?
+      false
+    end
+
+    def ==(other)
+      if other.is_a? ::Ci::StaticModel
+        id == other.id
+      else
+        super
+      end
+    end
+  end
+end
diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb
new file mode 100644
index 00000000000..2a87c91db5e
--- /dev/null
+++ b/lib/ci/version_info.rb
@@ -0,0 +1,52 @@
+class VersionInfo
+  include Comparable
+
+  attr_reader :major, :minor, :patch
+
+  def self.parse(str)
+    if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/)
+      VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i)
+    else
+      VersionInfo.new
+    end
+  end
+
+  def initialize(major = 0, minor = 0, patch = 0)
+    @major = major
+    @minor = minor
+    @patch = patch
+  end
+
+  def <=>(other)
+    return unless other.is_a? VersionInfo
+    return unless valid? && other.valid?
+
+    if other.major < @major
+      1
+    elsif @major < other.major
+      -1
+    elsif other.minor < @minor
+      1
+    elsif @minor < other.minor
+      -1
+    elsif other.patch < @patch
+      1
+    elsif @patch < other.patch
+      -1
+    else
+      0
+    end
+  end
+
+  def to_s
+    if valid?
+      "%d.%d.%d" % [@major, @minor, @patch]
+    else
+      "Unknown"
+    end
+  end
+
+  def valid?
+    @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0
+  end
+end