diff options
| -rw-r--r-- | lib/api/entities.rb | 38 | ||||
| -rw-r--r-- | lib/api/helpers/runner.rb | 28 | ||||
| -rw-r--r-- | lib/api/runner.rb | 39 | ||||
| -rw-r--r-- | spec/requests/api/runner_spec.rb | 257 |
4 files changed, 362 insertions, 0 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9dccaff369e..cb3b409b8d0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -697,5 +697,43 @@ module API expose :id, :message, :starts_at, :ends_at, :color, :font expose :active?, as: :active end + + class ArtifactFile < Grape::Entity + expose :filename, :size + end + + class JobCredentials < Grape::Entity + expose :type, :url, :username, :password + end + + class JobResponse < Grape::Entity + expose :id, :ref, :tag, :sha, :status + expose :name, :token, :stage + expose :project_id + expose :project_name + expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } + end + + class RequestJobResponse < JobResponse + expose :commands + expose :repo_url + expose :before_sha + expose :allow_git_fetch + expose :token + expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } + + expose :options do |model| + model.options + end + + expose :timeout do |model| + model.timeout + end + + expose :variables + expose :depends_on_builds, using: JobResponse + + expose :credentials, using: JobCredentials + end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 119ca81b883..8db0fc117c6 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,6 +1,8 @@ module API module Helpers module Runner + UPDATE_RUNNER_EVERY = 10 * 60 + def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], current_application_settings.runners_registration_token) @@ -18,6 +20,32 @@ module API def current_runner @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) end + + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age + end + + def build_not_found! + if headers['User-Agent'].to_s.match(/gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + no_content! + else + not_found! + end + end end end end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 47858f1866b..2e7b96e5169 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -48,5 +48,44 @@ module API Ci::Runner.find_by_token(params[:token]).destroy end end + + resource :jobs do + desc 'Request a job' do + success Entities::RequestJobResponse + end + params do + requires :token, type: String, desc: %q(Runner's authentication token) + end + post '/request' do + authenticate_runner! + not_found! unless current_runner.active? + update_runner_info + + if current_runner.is_runner_queue_value_latest?(params[:last_update]) + header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) + return build_not_found! + end + + new_update = current_runner.ensure_runner_queue_value + result = ::Ci::RegisterBuildService.new(current_runner).execute + + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) + present result.build, with: Entities::RequestJobResponse + else + Gitlab::Metrics.add_event(:build_not_found) + header 'X-GitLab-Last-Update', new_update + build_not_found! + end + else + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! + end + end + end end end diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index e83202e4196..3b94be55974 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -148,4 +148,261 @@ describe API::Runner do end end end + + describe '/api/v4/jobs' do + let(:project) { create(:empty_project, shared_runners_enabled: false) } + let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') } + let!(:job) { create(:ci_build, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0) } + let(:runner) { create(:ci_runner) } + + before { project.runners << runner } + + describe 'POST /api/v4/jobs/request' do + let!(:last_update) {} + let!(:new_update) { } + let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' } + + before { stub_container_registry_config(enabled: false) } + + shared_examples 'no jobs available' do + before { request_job } + + context 'when runner sends version in User-Agent' do + context 'for stable version' do + it 'gives 204 and set X-GitLab-Last-Update' do + expect(response).to have_http_status(204) + expect(response.header).to have_key('X-GitLab-Last-Update') + end + end + + context 'when last_update is up-to-date' do + let(:last_update) { runner.ensure_runner_queue_value } + + it 'gives 204 and set the same X-GitLab-Last-Update' do + expect(response).to have_http_status(204) + expect(response.header['X-GitLab-Last-Update']).to eq(last_update) + end + end + + context 'when last_update is outdated' do + let(:last_update) { runner.ensure_runner_queue_value } + let(:new_update) { runner.tick_runner_queue } + + it 'gives 204 and set a new X-GitLab-Last-Update' do + expect(response).to have_http_status(204) + expect(response.header['X-GitLab-Last-Update']).to eq(new_update) + end + end + + context 'for beta version' do + let(:user_agent) { 'gitlab-runner 9.0.0~beta.167.g2b2bacc (master; go1.7.4; linux/amd64)' } + it { expect(response).to have_http_status(204) } + end + + context 'for pre-9-0 version' do + let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0 (1-6-stable; go1.6.3; linux/amd64)' } + it { expect(response).to have_http_status(204) } + end + + context 'for pre-9-0 beta version' do + let(:user_agent) { 'gitlab-ci-multi-runner 1.6.0~beta.167.g2b2bacc (master; go1.6.3; linux/amd64)' } + it { expect(response).to have_http_status(204) } + end + end + + context %q(when runner doesn't send version in User-Agent) do + let(:user_agent) { 'Go-http-client/1.1' } + it { expect(response).to have_http_status(404) } + end + + context %q(when runner doesn't have a User-Agent) do + let(:user_agent) { nil } + it { expect(response).to have_http_status(404) } + end + end + + context 'when no token is provided' do + it 'returns 400 error' do + post api('/jobs/request') + expect(response).to have_http_status 400 + end + end + + context 'when invalid token is provided' do + it 'returns 403 error' do + post api('/jobs/request'), token: 'invalid' + expect(response).to have_http_status 403 + end + end + + context 'when valid token is provided' do + + context 'when Runner is not active' do + let(:runner) { create(:ci_runner, :inactive) } + + it 'returns 404 error' do + request_job + expect(response).to have_http_status 404 + end + end + + context 'when jobs are finished' do + before { job.success } + it_behaves_like 'no jobs available' + end + + context 'when other projects have pending jobs' do + before do + job.success + create(:ci_build, :pending) + end + + it_behaves_like 'no jobs available' + end + + context 'when shared runner requests job for project without shared_runners_enabled' do + let(:runner) { create(:ci_runner, :shared) } + it_behaves_like 'no jobs available' + end + + context 'when there is a pending job' do + it 'starts a job' do + request_job info: {platform: :darwin} + + expect(response).to have_http_status(201) + expect(response.headers).not_to have_key('X-GitLab-Last-Update') + expect(json_response['sha']).to eq(job.sha) + expect(json_response['options']).to eq({'image' => 'ruby:2.1', 'services' => ['postgres']}) + expect(json_response['variables']).to include( + {'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true}, + {'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true}, + {'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true} + ) + expect(runner.reload.platform).to eq('darwin') + end + + it 'updates runner info' do + expect { request_job }.to change { runner.reload.contacted_at } + end + + %w(name version revision platform architecture).each do |param| + context "when info parameter '#{param}' is present" do + let(:value) { "#{param}_value" } + + it %q(updates provided Runner's parameter) do + request_job info: {param => value} + + expect(response).to have_http_status(201) + runner.reload + expect(runner.read_attribute(param.to_sym)).to eq(value) + end + end + end + + context 'when concurrently updating a job' do + before do + expect_any_instance_of(Ci::Build).to receive(:run!). + and_raise(ActiveRecord::StaleObjectError.new(nil, nil)) + end + + it 'returns a conflict' do + request_job + expect(response).to have_http_status(409) + expect(response.headers).not_to have_key('X-GitLab-Last-Update') + end + end + + context 'when project and pipeline have multiple jobs' do + let!(:test_job) { create(:ci_build, pipeline: pipeline, name: 'deploy', stage: 'deploy', stage_idx: 1) } + + before { job.success } + + it 'returns dependent jobs' do + request_job + + expect(response).to have_http_status(201) + expect(json_response['id']).to eq(test_job.id) + expect(json_response['depends_on_builds'].count).to eq(1) + expect(json_response['depends_on_builds'][0]).to include('id' => job.id, 'name' => 'spinach') + end + end + + context 'when job has no tags' do + before { job.update(tags: []) } + + context 'when runner is allowed to pick untagged jobs' do + before { runner.update_column(:run_untagged, true) } + + it 'picks job' do + request_job + expect(response).to have_http_status 201 + end + end + + context 'when runner is not allowed to pick untagged jobs' do + before { runner.update_column(:run_untagged, false) } + it_behaves_like 'no jobs available' + end + end + + context 'when triggered job is available' do + before do + trigger = create(:ci_trigger, project: project) + create(:ci_trigger_request_with_variables, pipeline: pipeline, builds: [job], trigger: trigger) + project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value') + end + + it 'returns variables for triggers' do + request_job + + expect(response).to have_http_status(201) + expect(json_response['variables']).to include( + {'key' => 'CI_BUILD_NAME', 'value' => 'spinach', 'public' => true}, + {'key' => 'CI_BUILD_STAGE', 'value' => 'test', 'public' => true}, + {'key' => 'CI_BUILD_TRIGGERED', 'value' => 'true', 'public' => true}, + {'key' => 'DB_NAME', 'value' => 'postgres', 'public' => true}, + {'key' => 'SECRET_KEY', 'value' => 'secret_value', 'public' => false}, + {'key' => 'TRIGGER_KEY_1', 'value' => 'TRIGGER_VALUE_1', 'public' => false}, + ) + end + end + + describe 'registry credentials support' do + let(:registry_url) { 'registry.example.com:5005' } + let(:registry_credentials) do + {'type' => 'registry', + 'url' => registry_url, + 'username' => 'gitlab-ci-token', + 'password' => job.token} + end + + context 'when registry is enabled' do + before { stub_container_registry_config(enabled: true, host_port: registry_url) } + + it 'sends registry credentials key' do + request_job + expect(json_response).to have_key('credentials') + expect(json_response['credentials']).to include(registry_credentials) + end + end + + context 'when registry is disabled' do + before { stub_container_registry_config(enabled: false, host_port: registry_url) } + + it 'does not send registry credentials' do + request_job + expect(json_response).to have_key('credentials') + expect(json_response['credentials']).not_to include(registry_credentials) + end + end + end + end + + def request_job(token = runner.token, **params) + new_params = params.merge(token: token, last_update: last_update) + post api('/jobs/request'), new_params, {'User-Agent' => user_agent} + end + end + end + end end |