From 223c64f8b5b892a81fe438ad02fa9df264172f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 20 Apr 2016 10:29:03 +0000 Subject: Merge branch 'feature/incremental-build-trace-update' into 'master' Add incremental build trace update API Closes #14537 Must be merged if we want changes from gitlab-org/gitlab-ci-multi-runner!124 to work. See merge request !3447 --- CHANGELOG | 1 + app/models/ci/build.rb | 23 +++++++- doc/ci/api/builds.md | 108 +++++++++++++++++++++++++++++------- lib/ci/api/api.rb | 2 + lib/ci/api/builds.rb | 33 +++++++++++ spec/requests/ci/api/builds_spec.rb | 46 +++++++++++++++ 6 files changed, 191 insertions(+), 22 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index b42d9da70be..43775ed05c5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -102,6 +102,7 @@ v 8.7.0 (unreleased) - Allow enable/disable push events for system hooks - Fix GitHub project's link in the import page when provider has a custom URL - Add RAW build trace output and button on build page + - Add incremental build trace update into CI API v 8.6.7 (unreleased) - Fix vulnerability that made it possible to enumerate private projects belonging to group diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 85ef0523b31..d42a65620ff 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -230,12 +230,33 @@ module Ci end end + def trace_length + if raw_trace + raw_trace.length + else + 0 + end + end + def trace=(trace) + recreate_trace_dir + File.write(path_to_trace, trace) + end + + def recreate_trace_dir unless Dir.exists?(dir_to_trace) FileUtils.mkdir_p(dir_to_trace) end + end + private :recreate_trace_dir - File.write(path_to_trace, trace) + def append_trace(trace_part, offset) + recreate_trace_dir + + File.truncate(path_to_trace, offset) if File.exist?(path_to_trace) + File.open(path_to_trace, 'a') do |f| + f.write(trace_part) + end end def dir_to_trace diff --git a/doc/ci/api/builds.md b/doc/ci/api/builds.md index d100e261178..79761a893da 100644 --- a/doc/ci/api/builds.md +++ b/doc/ci/api/builds.md @@ -26,48 +26,114 @@ This API uses two types of authentication: ### Runs oldest pending build by runner - POST /ci/api/v1/builds/register +``` +POST /ci/api/v1/builds/register +``` -Parameters: +| Attribute | Type | Required | Description | +|-----------|---------|----------|---------------------| +| `token` | string | yes | Unique runner token | - * `token` (required) - Unique runner token +``` +curl -X POST "https://gitlab.example.com/ci/api/v1/builds/register" -F "token=t0k3n" +``` ### Update details of an existing build - PUT /ci/api/v1/builds/:id +``` +PUT /ci/api/v1/builds/:id +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|----------------------| +| `id` | integer | yes | The ID of a project | +| `token` | string | yes | Unique runner token | +| `state` | string | no | The state of a build | +| `trace` | string | no | The trace of a build | + +``` +curl -X PUT "https://gitlab.example.com/ci/api/v1/builds/1234" -F "token=t0k3n" -F "state=running" -F "trace=Running git clone...\n" +``` + +### Incremental build trace update + +Using this method you need to send trace content as a request body. You also need to provide the `Content-Range` header +with a range of sent trace part. Note that you need to send parts in the proper order, so the begining of the part +must start just after the end of the previous part. If you provide the wrong part, then GitLab CI API will return `416 +Range Not Satisfiable` response with a header `Range: 0-X`, where `X` is the current trace length. + +For example, if you receive `Range: 0-11` in the response, then your next part must contain a `Content-Range: 11-...` +header and a trace part covered by this range. + +For a valid update API will return `202` response with: +* `Build-Status: {status}` header containing current status of the build, +* `Range: 0-{length}` header with the current trace length. + +``` +PATCH /ci/api/v1/builds/:id/trace.txt +``` Parameters: - * `id` (required) - The ID of a project - * `token` (required) - Unique runner token - * `state` (optional) - The state of a build - * `trace` (optional) - The trace of a build +| Attribute | Type | Required | Description | +|-----------|---------|----------|----------------------| +| `id` | integer | yes | The ID of a build | + +Headers: + +| Attribute | Type | Required | Description | +|-----------------|---------|----------|-----------------------------------| +| `BUILD-TOKEN` | string | yes | The build authorization token | +| `Content-Range` | string | yes | Bytes range of trace that is sent | + +``` +curl -X PATCH "https://gitlab.example.com/ci/api/v1/builds/1234/trace.txt" -H "BUILD-TOKEN=build_t0k3n" -H "Content-Range=0-21" -d "Running git clone...\n" +``` + ### Upload artifacts to build - POST /ci/api/v1/builds/:id/artifacts +``` +POST /ci/api/v1/builds/:id/artifacts +``` -Parameters: +| Attribute | Type | Required | Description | +|-----------|---------|----------|-------------------------------| +| `id` | integer | yes | The ID of a build | +| `token` | string | yes | The build authorization token | +| `file` | mixed | yes | Artifacts file | - * `id` (required) - The ID of a build - * `token` (required) - The build authorization token - * `file` (required) - Artifacts file +``` +curl -X POST "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" -F "file=@/path/to/file" +``` ### Download the artifacts file from build - GET /ci/api/v1/builds/:id/artifacts +``` +GET /ci/api/v1/builds/:id/artifacts +``` -Parameters: +| Attribute | Type | Required | Description | +|-----------|---------|----------|-------------------------------| +| `id` | integer | yes | The ID of a build | +| `token` | string | yes | The build authorization token | - * `id` (required) - The ID of a build - * `token` (required) - The build authorization token +``` +curl "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" +``` ### Remove the artifacts file from build - DELETE /ci/api/v1/builds/:id/artifacts +``` +DELETE /ci/api/v1/builds/:id/artifacts +``` -Parameters: +| Attribute | Type | Required | Description | +|-----------|---------|----------|-------------------------------| +| ` id` | integer | yes | The ID of a build | +| `token` | string | yes | The build authorization token | - * ` id` (required) - The ID of a build - * `token` (required) - The build authorization token +``` +curl -X DELETE "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" +``` diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 4e85d2c3c74..353c4ddebf8 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -23,6 +23,8 @@ module Ci rack_response({ 'message' => '500 Internal Server Error' }, 500) end + content_type :txt, 'text/plain' + content_type :json, 'application/json' format :json helpers ::Ci::API::Helpers diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 2e9a5d311f9..607359769d1 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -50,6 +50,39 @@ module Ci end end + # Send incremental log update - Runners only + # + # Parameters: + # id (required) - The ID of a build + # Body: + # content of logs to append + # Headers: + # Content-Range (required) - range of content that was sent + # BUILD-TOKEN (required) - The build authorization token + # Example Request: + # PATCH /builds/:id/trace.txt + patch ":id/trace.txt" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('Build has been erased!') if build.erased? + + error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + content_range = request.headers['Content-Range'] + content_range = content_range.split('-') + + current_length = build.trace_length + unless current_length == content_range[0].to_i + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + end + + build.append_trace(request.body.read, content_range[0].to_i) + + status 202 + header 'Build-Status', build.status + header 'Range', "0-#{build.trace_length}" + end + # Authorize artifacts uploading for build - Runners only # # Parameters: diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb index 57d7eb927fd..ebd16c7efbe 100644 --- a/spec/requests/ci/api/builds_spec.rb +++ b/spec/requests/ci/api/builds_spec.rb @@ -156,6 +156,52 @@ describe Ci::API::API do end end + describe 'PATCH /builds/:id/trace.txt' do + let(:build) { create(:ci_build, :trace, runner_id: runner.id) } + let(:headers) { { Ci::API::Helpers::BUILD_TOKEN_HEADER => build.token, 'Content-Type' => 'text/plain' } } + let(:headers_with_range) { headers.merge({ 'Content-Range' => '11-20' }) } + + before do + build.run! + patch ci_api("/builds/#{build.id}/trace.txt"), ' appended', headers_with_range + end + + context 'when request is valid' do + it { expect(response.status).to eq 202 } + it { expect(build.reload.trace).to eq 'BUILD TRACE appended' } + it { expect(response.header).to have_key 'Range' } + it { expect(response.header).to have_key 'Build-Status' } + end + + context 'when content-range start is too big' do + let(:headers_with_range) { headers.merge({ 'Content-Range' => '15-20' }) } + + it { expect(response.status).to eq 416 } + it { expect(response.header).to have_key 'Range' } + it { expect(response.header['Range']).to eq '0-11' } + end + + context 'when content-range start is too small' do + let(:headers_with_range) { headers.merge({ 'Content-Range' => '8-20' }) } + + it { expect(response.status).to eq 416 } + it { expect(response.header).to have_key 'Range' } + it { expect(response.header['Range']).to eq '0-11' } + end + + context 'when Content-Range header is missing' do + let(:headers_with_range) { headers.merge({}) } + + it { expect(response.status).to eq 400 } + end + + context 'when build has been errased' do + let(:build) { create(:ci_build, runner_id: runner.id, erased_at: Time.now) } + + it { expect(response.status).to eq 403 } + end + end + context "Artifacts" do let(:file_upload) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') } let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') } -- cgit v1.2.1