diff options
| -rw-r--r-- | Gemfile | 5 | ||||
| -rw-r--r-- | Gemfile.lock | 27 | ||||
| -rw-r--r-- | VERSION | 2 | ||||
| -rw-r--r-- | app/controllers/projects_controller.rb | 4 | ||||
| -rw-r--r-- | app/models/build.rb | 31 | ||||
| -rw-r--r-- | config/routes.rb | 4 | ||||
| -rw-r--r-- | lib/api/api.rb | 28 | ||||
| -rw-r--r-- | lib/api/builds.rb | 45 | ||||
| -rw-r--r-- | lib/api/entities.rb | 7 | ||||
| -rw-r--r-- | lib/api/helpers.rb | 104 | ||||
| -rw-r--r-- | lib/gitlab_ci/encode.rb | 33 | ||||
| -rw-r--r-- | lib/runner.rb | 117 | ||||
| -rw-r--r-- | lib/scheduler.rb | 3 |
13 files changed, 226 insertions, 184 deletions
@@ -49,11 +49,14 @@ gem 'state_machine' # Encoding detection gem 'charlock_holmes' +# API +gem 'grape' +gem 'grape-entity' + # Other gem 'rake' gem 'foreman' gem 'jquery-rails' -gem 'childprocess', '0.3.6' gem 'gitlab_ci_meta' group :assets do diff --git a/Gemfile.lock b/Gemfile.lock index 12b81a0..b26d19e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -31,6 +31,7 @@ GEM annotate (2.5.0) rake arel (3.0.2) + backports (2.6.7) bcrypt-ruby (3.0.1) bootstrap-sass (2.3.1.0) sass (~> 3.2) @@ -65,6 +66,7 @@ GEM rest-client simplecov (>= 0.7) thor + descendants_tracker (0.0.1) devise (2.2.3) bcrypt-ruby (~> 3.0) orm_adapter (~> 0.1) @@ -88,6 +90,19 @@ GEM foreman (0.62.0) thor (>= 0.13.6) gitlab_ci_meta (1.0) + grape (0.4.1) + activesupport + builder + hashie (>= 1.2.0) + multi_json (>= 1.3.2) + multi_xml (>= 0.5.2) + rack (>= 1.3.0) + rack-accept + rack-mount + virtus + grape-entity (0.3.0) + activesupport + multi_json (>= 1.3.2) growl (1.0.3) guard (1.3.2) listen (>= 0.4.2) @@ -100,6 +115,7 @@ GEM activesupport (>= 3.1, < 4.1) haml (~> 3.1) railties (>= 3.1, < 4.1) + hashie (2.0.5) hike (1.2.2) i18n (0.6.1) journey (1.0.4) @@ -119,6 +135,7 @@ GEM method_source (0.8.1) mime-types (1.22) multi_json (1.7.2) + multi_xml (0.5.3) mysql2 (0.3.11) nokogiri (1.5.9) orm_adapter (0.4.0) @@ -133,8 +150,12 @@ GEM quiet_assets (1.0.1) railties (~> 3.1) rack (1.4.5) + rack-accept (0.4.5) + rack (>= 0.4) rack-cache (1.2) rack (>= 0.4) + rack-mount (0.8.3) + rack (>= 1.0.0) rack-protection (1.2.0) rack rack-ssl (1.3.3) @@ -233,6 +254,9 @@ GEM uglifier (1.3.0) execjs (>= 0.3.0) multi_json (~> 1.0, >= 1.0.2) + virtus (0.5.4) + backports (~> 2.6.1) + descendants_tracker (~> 0.0.1) warden (1.2.1) rack (>= 1.0) websocket (1.0.7) @@ -250,7 +274,6 @@ DEPENDENCIES bootstrap-sass capybara charlock_holmes - childprocess (= 0.3.6) coffee-rails (~> 3.2.1) coveralls devise @@ -259,6 +282,8 @@ DEPENDENCIES font-awesome-sass-rails (~> 3.0.0) foreman gitlab_ci_meta + grape + grape-entity growl guard-rspec haml-rails @@ -1 +1 @@ -2.2.0 +3.0.0.pre diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index abfe1dd..8d8ac5c 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -1,5 +1,3 @@ -require 'runner' - class ProjectsController < ApplicationController before_filter :authenticate_user!, except: [:build, :status, :index, :show] before_filter :project, only: [:build, :details, :show, :status, :edit, :update, :destroy, :stats] @@ -64,7 +62,6 @@ class ProjectsController < ApplicationController @build = @project.register_build(ref: params[:ref]) if @build and @build.id - Runner.perform_async(@build.id) unless @build.ci_skip? redirect_to project_build_path(@project, @build) else redirect_to project_path(@project), notice: 'Branch is not defined for this project' @@ -85,7 +82,6 @@ class ProjectsController < ApplicationController @build = @project.register_build(build_params) if @build - Runner.perform_async(@build.id) head 200 else head 500 diff --git a/app/models/build.rb b/app/models/build.rb index 4f4d17f..de88ff5 100644 --- a/app/models/build.rb +++ b/app/models/build.rb @@ -78,11 +78,6 @@ class Build < ActiveRecord::Base @commit ||= project.last_commit(self.sha) end - def write_trace(trace) - self.reload - update_attributes(trace: trace) - end - def short_before_sha before_sha[0..8] end @@ -92,25 +87,10 @@ class Build < ActiveRecord::Base end def trace_html - html = Ansi2html::convert(compose_output) if trace.present? + html = Ansi2html::convert(trace) if trace.present? html ||= '' end - def read_tmp_file - content = GitlabCi::Encode.encode!(File.binread(tmp_file)) if tmp_file && File.readable?(tmp_file) - content ||= '' - end - - def compose_output - output = trace - - if running? - output << read_tmp_file - end - - output - end - def to_param sha end @@ -123,9 +103,12 @@ class Build < ActiveRecord::Base running? || pending? end - def set_file path - self.tmp_file = path - self.save + def commands + project.scripts + end + + def path + project.path end end diff --git a/config/routes.rb b/config/routes.rb index c5e2cce..2877e16 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -7,6 +7,10 @@ GitlabCi::Application.routes.draw do mount Sidekiq::Web, at: "/ext/sidekiq", as: :ext_resque end + # API + API::API.logger Rails.logger + mount API::API => '/api' + resources :projects do member do get :run diff --git a/lib/api/api.rb b/lib/api/api.rb new file mode 100644 index 0000000..4fc1926 --- /dev/null +++ b/lib/api/api.rb @@ -0,0 +1,28 @@ +Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} + +module API + class API < Grape::API + version 'v1', using: :path + + rescue_from ActiveRecord::RecordNotFound do + rack_response({'message' => '404 Not found'}.to_json, 404) + end + + rescue_from :all do |exception| + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 + # why is this not wrapped in something reusable? + trace = exception.backtrace + + message = "\n#{exception.class} (#{exception.message}):\n" + message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) + message << " " << trace.join("\n ") + + API.logger.add Logger::FATAL, message + rack_response({'message' => '500 Internal Server Error'}, 500) + end + + format :json + helpers Helpers + mount Builds + end +end diff --git a/lib/api/builds.rb b/lib/api/builds.rb new file mode 100644 index 0000000..2c77001 --- /dev/null +++ b/lib/api/builds.rb @@ -0,0 +1,45 @@ +module API + # Issues API + class Builds < Grape::API + resource :builds do + # Register a build by runner + # + # Parameters: + # token (required) - The uniq token of runner + # + # Example Request: + # POST /builds/register + post "register" do + required_attributes! [:token] + + ActiveRecord::Base.transaction do + build = Build.pending.order('created_at ASC').first + not_found! and return unless build + + build.run! + present build, with: Entities::Build + end + end + + # Update an existing build + # + # Parameters: + # id (required) - The ID of a project + # state (optional) - The state of a build + # output (optional) - The trace of a build + # Example Request: + # PUT /builds/:id + put ":id" do + build = Build.find(params[:id]) + build.update_attributes trace: params[:trace] + + case params[:state].to_s + when 'success' + build.success + when 'failed' + build.drop + end + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb new file mode 100644 index 0000000..fdbc29b --- /dev/null +++ b/lib/api/entities.rb @@ -0,0 +1,7 @@ +module API + module Entities + class Build < Grape::Entity + expose :id, :commands, :path, :ref, :sha + end + end +end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb new file mode 100644 index 0000000..4448227 --- /dev/null +++ b/lib/api/helpers.rb @@ -0,0 +1,104 @@ +module API + module Helpers + def current_user + @current_user ||= User.find_by_authentication_token(params[:private_token] || env["HTTP_PRIVATE_TOKEN"]) + end + + def user_project + @project ||= find_project + @project || not_found! + end + + def find_project + project = Project.find_by_id(params[:id]) || Project.find_with_namespace(params[:id]) + + if project && can?(current_user, :read_project, project) + project + else + nil + end + end + + def paginate(object) + object.page(params[:page]).per(params[:per_page].to_i) + end + + def authenticate! + unauthorized! unless current_user + end + + def authenticated_as_admin! + forbidden! unless current_user.is_admin? + end + + def authorize! action, subject + unless abilities.allowed?(current_user, action, subject) + forbidden! + end + end + + def can?(object, action, subject) + abilities.allowed?(object, action, subject) + end + + # Checks the occurrences of required attributes, each attribute must be present in the params hash + # or a Bad Request error is invoked. + # + # Parameters: + # keys (required) - A hash consisting of keys that must be present + def required_attributes!(keys) + keys.each do |key| + bad_request!(key) unless params[key].present? + end + end + + def attributes_for_keys(keys) + attrs = {} + keys.each do |key| + attrs[key] = params[key] if params[key].present? + end + attrs + end + + # error helpers + + def forbidden! + render_api_error!('403 Forbidden', 403) + end + + def bad_request!(attribute) + message = ["400 (Bad request)"] + message << "\"" + attribute.to_s + "\" not given" + render_api_error!(message.join(' '), 400) + end + + def not_found!(resource = nil) + message = ["404"] + message << resource if resource + message << "Not Found" + render_api_error!(message.join(' '), 404) + end + + def unauthorized! + render_api_error!('401 Unauthorized', 401) + end + + def not_allowed! + render_api_error!('Method Not Allowed', 405) + end + + def render_api_error!(message, status) + error!({'message' => message}, status) + end + + private + + def abilities + @abilities ||= begin + abilities = Six.new + abilities << Ability + abilities + end + end + end +end diff --git a/lib/gitlab_ci/encode.rb b/lib/gitlab_ci/encode.rb deleted file mode 100644 index 41b8a66..0000000 --- a/lib/gitlab_ci/encode.rb +++ /dev/null @@ -1,33 +0,0 @@ -require 'charlock_holmes/string' - -module GitlabCi - module Encode - extend self - - def encode!(message) - return nil unless message.respond_to? :force_encoding - - # if message is utf-8 encoding, just return it - message.force_encoding("UTF-8") - return message if message.valid_encoding? - - # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) - return message if detect[:type] == :binary - - # if message is not utf-8 encoding, convert it - if detect[:encoding] - message.force_encoding(detect[:encoding]) - message.encode!("UTF-8", detect[:encoding], undef: :replace, replace: "", invalid: :replace) - end - - # ensure message encoding is utf8 - message.valid_encoding? ? message : raise - - # Prevent app from crash cause of encoding errors - rescue - encoding = detect ? detect[:encoding] : "unknown" - "--broken encoding: #{encoding}" - end - end -end diff --git a/lib/runner.rb b/lib/runner.rb deleted file mode 100644 index 59e0375..0000000 --- a/lib/runner.rb +++ /dev/null @@ -1,117 +0,0 @@ -class Runner - include Sidekiq::Worker - - attr_accessor :project, :build, :output - - sidekiq_options queue: :runner - - def perform(build_id) - @build = Build.find(build_id) - @project = @build.project - @output = '' - - return true if @build.canceled? - - run_in_transaction ? run : run_later - end - - def run_in_transaction - ActiveRecord::Base.transaction do - build.run! if project.no_running_builds? - end - end - - def run_later - Runner.perform_in(2.minutes, @build.id) - end - - def initialize - @logger = Logger.new(STDOUT) - @logger.level = Logger::INFO - end - - def run - path = project.path - commands = project.scripts - commands = commands.lines.to_a - commands.unshift(prepare_project_cmd(path, build.sha)) - - commands.each do |line| - status = command(line, path) - build.write_trace(@output) - - return if build.canceled? - - unless status - build.drop! - return - end - end - - build.success! - ensure - build.write_trace(@output) - end - - def command(cmd, path) - cmd = cmd.strip - status = 0 - - @output ||= "" - @output << "\n" - @output << cmd - @output << "\n" - - @process = ChildProcess.build(cmd) - @tmp_file = Tempfile.new("child-output", binmode: true) - @process.io.stdout = @tmp_file - @process.io.stderr = @tmp_file - @process.cwd = path - - # ENV - @process.environment['BUNDLE_GEMFILE'] = File.join(path, 'Gemfile') - @process.environment['BUNDLE_BIN_PATH'] = '' - @process.environment['RUBYOPT'] = '' - - @process.environment['CI_SERVER'] = 'yes' - @process.environment['CI_SERVER_NAME'] = 'GitLab CI' - @process.environment['CI_SERVER_VERSION'] = GitlabCi::Version - @process.environment['CI_SERVER_REVISION'] = GitlabCi::Revision - - @process.environment['CI_BUILD_REF'] = build.ref - - @process.start - - build.set_file @tmp_file.path - - begin - @process.poll_for_exit(project.timeout) - rescue ChildProcess::TimeoutError - @output << "TIMEOUT" - @process.stop # tries increasingly harsher methods to kill the process. - return false - end - - @process.exit_code == 0 - - rescue => e - # return false if any exception occurs - @output << e.message - false - - ensure - @tmp_file.rewind - @output << GitlabCi::Encode.encode!(@tmp_file.read) - @tmp_file.close - @tmp_file.unlink - end - - def prepare_project_cmd(path, ref) - cmd = [] - cmd << "cd #{path}" - cmd << "git fetch" - cmd << "git reset --hard" - cmd << "git checkout #{ref}" - cmd.join(" && ") - end -end diff --git a/lib/scheduler.rb b/lib/scheduler.rb index ec4ed2c..0418df0 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -6,9 +6,6 @@ class Scheduler interval = project.polling_interval if (last_build_time + interval.hours) < Time.now build = project.register_build(ref: project.tracked_refs.first) - if build and build.id - Runner.perform_async(build.id) - end end end end |