summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKamil Trzcinski <ayufan@ayufan.eu>2015-10-12 23:47:32 +0200
committerKamil Trzcinski <ayufan@ayufan.eu>2015-11-10 12:51:50 +0100
commitd0e3e823a2dd56260550aec648b0cbfae64543ae (patch)
tree22939b81b27610b602c6714afff83cd54781eeda
parent354b69dde2ba399a4269a0f544fd7a4e399d8b7e (diff)
downloadgitlab-ce-d0e3e823a2dd56260550aec648b0cbfae64543ae.tar.gz
Implement Build Artifacts
- Offloads uploading to GitLab Workhorse - Use /authorize request for fast uploading - Added backup recipes for artifacts - Support download acceleration using X-Sendfile
-rw-r--r--.gitignore3
-rw-r--r--CHANGELOG1
-rw-r--r--Gemfile2
-rw-r--r--Gemfile.lock6
-rw-r--r--app/controllers/admin/application_settings_controller.rb1
-rw-r--r--app/controllers/projects/builds_controller.rb27
-rw-r--r--app/models/ability.rb1
-rw-r--r--app/models/application_setting.rb1
-rw-r--r--app/models/ci/build.rb17
-rw-r--r--app/models/commit_status.rb4
-rw-r--r--app/uploaders/artifact_uploader.rb50
-rw-r--r--app/views/admin/application_settings/_form.html.haml5
-rw-r--r--app/views/projects/builds/show.html.haml3
-rw-r--r--app/views/projects/commit_statuses/_commit_status.html.haml3
-rw-r--r--config/initializers/1_settings.rb1
-rw-r--r--config/routes.rb1
-rw-r--r--db/migrate/20151013092124_add_artifacts_file_to_builds.rb5
-rw-r--r--db/migrate/20151109100728_add_max_artifacts_size_to_application_settings.rb5
-rw-r--r--db/schema.rb4
-rw-r--r--doc/ci/yaml/README.md15
-rw-r--r--doc/install/installation.md5
-rw-r--r--doc/raketasks/backup_restore.md3
-rw-r--r--lib/api/helpers.rb44
-rw-r--r--lib/backup/artifacts.rb13
-rw-r--r--lib/backup/manager.rb2
-rw-r--r--lib/ci/api/api.rb1
-rw-r--r--lib/ci/api/builds.rb102
-rw-r--r--lib/ci/api/entities.rb7
-rw-r--r--lib/ci/api/helpers.rb11
-rw-r--r--lib/ci/gitlab_ci_yaml_processor.rb9
-rw-r--r--lib/file_streamer.rb16
-rw-r--r--lib/gitlab/current_settings.rb1
-rw-r--r--lib/support/nginx/gitlab16
-rw-r--r--lib/support/nginx/gitlab-ssl16
-rw-r--r--lib/tasks/gitlab/backup.rake21
-rw-r--r--lib/uploaded_file.rb37
-rw-r--r--shared/artifacts/.gitkeep0
-rw-r--r--shared/tmp/artifacts-cache/.gitkeep0
-rw-r--r--shared/tmp/artifacts-uploads/.gitkeep0
-rw-r--r--spec/features/builds_spec.rb21
-rw-r--r--spec/features/commits_spec.rb16
-rw-r--r--spec/lib/ci/gitlab_ci_yaml_processor_spec.rb38
-rw-r--r--spec/models/build_spec.rb15
-rw-r--r--spec/requests/api/users_spec.rb15
-rw-r--r--spec/requests/ci/api/builds_spec.rb186
-rw-r--r--spec/tasks/gitlab/backup_rake_spec.rb14
46 files changed, 740 insertions, 24 deletions
diff --git a/.gitignore b/.gitignore
index 73bde4cc761..fd137a50473 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,6 +37,9 @@ nohup.out
public/assets/
public/uploads.*
public/uploads/
+shared/artifacts/
+shared/tmp/artifacts-uploads/
+shared/tmp/artifacts-cache/
rails_best_practices_output.html
/tags
tmp/
diff --git a/CHANGELOG b/CHANGELOG
index d11076f4848..e54e7329ca1 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -95,6 +95,7 @@ v 8.1.0 (unreleased)
- Show CI status on Your projects page and Starred projects page
- Remove "Continuous Integration" page from dashboard
- Add notes and SSL verification entries to hook APIs (Ben Boeckel)
+ - Added build artifacts
- Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
- Move CI runners page to project settings area
- Move CI variables page to project settings area
diff --git a/Gemfile b/Gemfile
index 51848621b64..49e35e873ba 100644
--- a/Gemfile
+++ b/Gemfile
@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2'
gem "github-linguist", "~> 4.7.0", require: "linguist"
# API
-gem 'grape', '~> 0.6.1'
+gem 'grape', '~> 0.13.0'
gem 'grape-entity', '~> 0.4.2'
gem 'rack-cors', '~> 0.4.0', require: 'rack/cors'
diff --git a/Gemfile.lock b/Gemfile.lock
index d358e236302..310694ff6af 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -306,10 +306,10 @@ GEM
gon (5.0.4)
actionpack (>= 2.3.0)
json
- grape (0.6.1)
+ grape (0.13.0)
activesupport
builder
- hashie (>= 1.2.0)
+ hashie (>= 2.1.0)
multi_json (>= 1.3.2)
multi_xml (>= 0.5.2)
rack (>= 1.3.0)
@@ -829,7 +829,7 @@ DEPENDENCIES
gitlab_omniauth-ldap (~> 1.2.1)
gollum-lib (~> 4.0.2)
gon (~> 5.0.0)
- grape (~> 0.6.1)
+ grape (~> 0.13.0)
grape-entity (~> 0.4.2)
haml-rails (~> 0.9.0)
hipchat (~> 1.5.0)
diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb
index 3d9c59050ff..a9bcfc7456a 100644
--- a/app/controllers/admin/application_settings_controller.rb
+++ b/app/controllers/admin/application_settings_controller.rb
@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:admin_notification_email,
:user_oauth_applications,
:shared_runners_enabled,
+ :max_artifacts_size,
restricted_visibility_levels: [],
import_sources: []
)
diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb
index 953f30e7c03..4638f77b887 100644
--- a/app/controllers/projects/builds_controller.rb
+++ b/app/controllers/projects/builds_controller.rb
@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
before_action :build, except: [:index, :cancel_all]
before_action :authorize_manage_builds!, except: [:index, :show, :status]
+ before_action :authorize_download_build_artifacts!, only: [:download]
layout "project"
@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
redirect_to build_path(build)
end
+ def download
+ unless artifacts_file.file_storage?
+ return redirect_to artifacts_file.url
+ end
+
+ unless artifacts_file.exists?
+ return not_found!
+ end
+
+ send_file artifacts_file.path, disposition: 'attachment'
+ end
+
def status
render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
end
@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
@build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
end
+ def artifacts_file
+ build.artifacts_file
+ end
+
def build_path(build)
namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
end
@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
return page_404
end
end
+
+ def authorize_download_build_artifacts!
+ unless can?(current_user, :download_build_artifacts, @project)
+ if current_user.nil?
+ return authenticate_user!
+ else
+ return render_404
+ end
+ end
+ end
end
diff --git a/app/models/ability.rb b/app/models/ability.rb
index b72178fa126..5ae28d5133e 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -154,6 +154,7 @@ class Ability
:create_merge_request,
:create_wiki,
:manage_builds,
+ :download_build_artifacts,
:push_code
]
end
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 266045f7afa..fa7cf2464ad 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+ max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
)
end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 7f185ae7cc3..0ec7e210321 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -39,6 +39,8 @@ module Ci
scope :ignore_failures, ->() { where(allow_failure: false) }
scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
+ mount_uploader :artifacts_file, ArtifactUploader
+
acts_as_taggable
# To prevent db load megabytes of data from trace
@@ -217,6 +219,14 @@ module Ci
"#{dir_to_trace}/#{id}.log"
end
+ def token
+ project.token
+ end
+
+ def valid_token? token
+ project.valid_token? token
+ end
+
def target_url
Gitlab::Application.routes.url_helpers.
namespace_project_build_url(gl_project.namespace, gl_project, self)
@@ -248,6 +258,13 @@ module Ci
pending? && !any_runners_online?
end
+ def download_url
+ if artifacts_file.exists?
+ Gitlab::Application.routes.url_helpers.
+ download_namespace_project_build_path(gl_project.namespace, gl_project, self)
+ end
+ end
+
private
def yaml_variables
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 7d54d83974a..d346c5d35d2 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
def show_warning?
false
end
+
+ def download_url
+ nil
+ end
end
diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb
new file mode 100644
index 00000000000..848e0bcbde1
--- /dev/null
+++ b/app/uploaders/artifact_uploader.rb
@@ -0,0 +1,50 @@
+# encoding: utf-8
+class ArtifactUploader < CarrierWave::Uploader::Base
+ storage :file
+
+ attr_accessor :build, :field
+
+ def self.artifacts_path
+ File.expand_path('shared/artifacts/', Rails.root)
+ end
+
+ def self.artifacts_upload_path
+ File.expand_path('shared/tmp/artifacts-uploads/', Rails.root)
+ end
+
+ def self.artifacts_cache_path
+ File.expand_path('shared/tmp/artifacts-cache/', Rails.root)
+ end
+
+ def initialize(build, field)
+ @build, @field = build, field
+ end
+
+ def artifacts_path
+ File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
+ end
+
+ def store_dir
+ File.join(ArtifactUploader.artifacts_path, artifacts_path)
+ end
+
+ def cache_dir
+ File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
+ end
+
+ def file_storage?
+ self.class.storage == CarrierWave::Storage::File
+ end
+
+ def exists?
+ file.try(:exists?)
+ end
+
+ def move_to_cache
+ true
+ end
+
+ def move_to_store
+ true
+ end
+end
diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml
index 7253218c2e9..ddaf0e0e8ff 100644
--- a/app/views/admin/application_settings/_form.html.haml
+++ b/app/views/admin/application_settings/_form.html.haml
@@ -139,5 +139,10 @@
= f.check_box :shared_runners_enabled
Enable shared runners for a new projects
+ .form-group
+ = f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
+ .col-sm-10
+ = f.number_field :max_artifacts_size, class: 'form-control'
+
.form-actions
= f.submit 'Save', class: 'btn btn-primary'
diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml
index 3374d5432a5..7661452e6ec 100644
--- a/app/views/projects/builds/show.html.haml
+++ b/app/views/projects/builds/show.html.haml
@@ -87,6 +87,9 @@
Test coverage
%h1 #{@build.coverage}%
+ - if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
+ .build-widget.center
+ = link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
.build-widget
%h4.title
diff --git a/app/views/projects/commit_statuses/_commit_status.html.haml b/app/views/projects/commit_statuses/_commit_status.html.haml
index c255559b88c..9a0e7bff3f1 100644
--- a/app/views/projects/commit_statuses/_commit_status.html.haml
+++ b/app/views/projects/commit_statuses/_commit_status.html.haml
@@ -61,6 +61,9 @@
%td
.pull-right
+ - if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
+ = link_to commit_status.download_url, title: 'Download artifacts' do
+ %i.fa.fa-download
- if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
- if commit_status.active?
- if commit_status.cancel_url
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 8192d727f2a..b39e263e39a 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_br
Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil?
Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url)
Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
+Settings.gitlab_ci['max_artifacts_size'] ||= 100
#
# Reply by email
diff --git a/config/routes.rb b/config/routes.rb
index 2028ea938e4..c892c034f01 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do
member do
get :status
post :cancel
+ get :download
post :retry
end
end
diff --git a/db/migrate/20151013092124_add_artifacts_file_to_builds.rb b/db/migrate/20151013092124_add_artifacts_file_to_builds.rb
new file mode 100644
index 00000000000..5a299f7b26d
--- /dev/null
+++ b/db/migrate/20151013092124_add_artifacts_file_to_builds.rb
@@ -0,0 +1,5 @@
+class AddArtifactsFileToBuilds < ActiveRecord::Migration
+ def change
+ add_column :ci_builds, :artifacts_file, :text
+ end
+end
diff --git a/db/migrate/20151109100728_add_max_artifacts_size_to_application_settings.rb b/db/migrate/20151109100728_add_max_artifacts_size_to_application_settings.rb
new file mode 100644
index 00000000000..01d8c0f043e
--- /dev/null
+++ b/db/migrate/20151109100728_add_max_artifacts_size_to_application_settings.rb
@@ -0,0 +1,5 @@
+class AddMaxArtifactsSizeToApplicationSettings < ActiveRecord::Migration
+ def change
+ add_column :application_settings, :max_artifacts_size, :integer, default: 100, null: false
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 116c0c8d97d..f631d73f334 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20151105094515) do
+ActiveRecord::Schema.define(version: 20151109100728) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.text "help_page_text"
t.string "admin_notification_email"
t.boolean "shared_runners_enabled", default: true, null: false
+ t.integer "max_artifacts_size", default: 100, null: false
end
create_table "audit_events", force: true do |t|
@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.string "type"
t.string "target_url"
t.string "description"
+ t.text "artifacts_file"
end
add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md
index d117a2969be..d8504aca86a 100644
--- a/doc/ci/yaml/README.md
+++ b/doc/ci/yaml/README.md
@@ -141,6 +141,7 @@ job_name:
| tags | optional | Defines a list of tags which are used to select runner |
| allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
| when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
+| artifacts | optional | Define list build artifacts |
### script
`script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
@@ -258,6 +259,20 @@ The above script will:
1. Execute `cleanup_build` only when the `build` failed,
2. Always execute `cleanup` as the last step in pipeline.
+### artifacts
+`artifacts` is used to specify list of files and directories which should be attached to build after success.
+
+```
+artifacts:
+- binaries/
+- .config
+```
+
+The above definition will archive all files in `binaries/` and `.config`.
+The artifacts will be send after the build success to GitLab and will be accessible in GitLab interface to download.
+
+This feature requires GitLab Runner v 0.7.0.
+
## Validate the .gitlab-ci.yml
Each instance of GitLab CI has an embedded debug tool called Lint.
You can find the link to the Lint in the project's settings page or use short url `/lint`.
diff --git a/doc/install/installation.md b/doc/install/installation.md
index f17477a3218..0ece8bd9315 100644
--- a/doc/install/installation.md
+++ b/doc/install/installation.md
@@ -246,6 +246,11 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
# Change the permissions of the directory where CI build traces are stored
sudo chmod -R u+rwX builds/
+ # Change the permissions of the directory where CI artifacts are stored
+ sudo chmod -R u+rwX shared/artifacts/
+ sudo chmod -R u+rwX shared/tmp/artifacts-uploads/
+ sudo chmod -R u+rwX shared/tmp/artifacts-cache/
+
# Copy the example Unicorn config
sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md
index 606532a6fbe..1a5442cdac7 100644
--- a/doc/raketasks/backup_restore.md
+++ b/doc/raketasks/backup_restore.md
@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
```
Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
-uploads (attachments), repositories, builds(CI build output logs). Use a comma to specify several options at the same time.
+uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts).
+Use a comma to specify several options at the same time.
```
sudo gitlab-rake gitlab:backup:create SKIP=db,uploads
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 652bdf9b278..b980cd8391e 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -133,6 +133,12 @@ module API
authorize! :admin_project, user_project
end
+ def require_gitlab_workhorse!
+ unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
+ forbidden!('Request should be executed via GitLab Workhorse')
+ end
+ end
+
def can?(object, action, subject)
abilities.allowed?(object, action, subject)
end
@@ -234,6 +240,10 @@ module API
render_api_error!(message || '409 Conflict', 409)
end
+ def file_to_large!
+ render_api_error!('413 Request Entity Too Large', 413)
+ end
+
def render_validation_error!(model)
if model.errors.any?
render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +292,40 @@ module API
end
end
+ # file helpers
+
+ def uploaded_file!(uploads_path)
+ required_attributes! [:file]
+
+ # sanitize file paths
+ # this requires for all paths to exist
+ uploads_path = File.realpath(uploads_path)
+ file_path = File.realpath(params[:file])
+ bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+
+ UploadedFile.new(
+ file_path,
+ params[:filename],
+ params[:filetype] || 'application/octet-stream',
+ )
+ end
+
+ def present_file!(path, filename, content_type = 'application/octet-stream')
+ filename ||= File.basename(path)
+ header['Content-Disposition'] = "attachment; filename=#{filename}"
+ header['Content-Transfer-Encoding'] = 'binary'
+ content_type content_type
+
+ # Support download acceleration
+ case headers['X-Sendfile-Type']
+ when 'X-Sendfile'
+ header['X-Sendfile'] = path
+ body
+ else
+ file FileStreamer.new(path)
+ end
+ end
+
private
def add_pagination_headers(paginated, per_page)
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
new file mode 100644
index 00000000000..51fa3867e67
--- /dev/null
+++ b/lib/backup/artifacts.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+ class Artifacts < Files
+ def initialize
+ super('artifacts', ArtifactUploader.artifacts_path)
+ end
+
+ def create_files_dir
+ Dir.mkdir(app_files_dir, 0700)
+ end
+ end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f011fd03de0..9e15d5411a1 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,7 +150,7 @@ module Backup
private
def backup_contents
- folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+ folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
end
def folders_to_backup
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 0a4cbf69b63..07e68216d7f 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -27,6 +27,7 @@ module Ci
helpers Helpers
helpers ::API::Helpers
+ helpers Gitlab::CurrentSettings
mount Builds
mount Commits
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 83ca1e6481c..622849c4b11 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -47,6 +47,108 @@ module Ci
build.drop
end
end
+
+ # Authorize artifacts uploading for build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # size (optional) - the size of uploaded file
+ # Example Request:
+ # POST /builds/:id/artifacts/authorize
+ post ":id/artifacts/authorize" do
+ require_gitlab_workhorse!
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ forbidden!('build is not running') unless build.running?
+
+ if params[:filesize]
+ file_size = params[:filesize].to_i
+ file_to_large! unless file_size < max_artifacts_size
+ end
+
+ status 200
+ { temp_path: ArtifactUploader.artifacts_upload_path }
+ end
+
+ # Upload artifacts to build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # Content-Type - File content type
+ # Content-Disposition - File media type and real name
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Body:
+ # The file content
+ #
+ # Parameters (set by GitLab Workhorse):
+ # file - path to locally stored body (generated by Workhorse)
+ # filename - real filename as send in Content-Disposition
+ # filetype - real content type as send in Content-Type
+ # filesize - real file size as send in Content-Length
+ # Example Request:
+ # POST /builds/:id/artifacts
+ post ":id/artifacts" do
+ require_gitlab_workhorse!
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ forbidden!('build is not running') unless build.running?
+
+ file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
+ file_to_large! unless file.size < max_artifacts_size
+
+ if build.update_attributes(artifacts_file: file)
+ present build, with: Entities::Build
+ else
+ render_validation_error!(build)
+ end
+ end
+
+ # Download the artifacts file from build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Example Request:
+ # GET /builds/:id/artifacts
+ get ":id/artifacts" do
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ artifacts_file = build.artifacts_file
+
+ unless artifacts_file.file_storage?
+ return redirect_to build.artifacts_file.url
+ end
+
+ unless artifacts_file.exists?
+ not_found!
+ end
+
+ present_file!(artifacts_file.path, artifacts_file.filename)
+ end
+
+ # Remove the artifacts file from build
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Example Request:
+ # DELETE /builds/:id/artifacts
+ delete ":id/artifacts" do
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ build.remove_artifacts_file!
+ end
end
end
end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b80c0b8b273..750f421872d 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -11,10 +11,16 @@ module Ci
expose :builds
end
+ class ArtifactFile < Grape::Entity
+ expose :filename, :size
+ end
+
class Build < Grape::Entity
expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
:before_sha, :allow_git_fetch, :project_name
+ expose :name, :token, :stage
+
expose :options do |model|
model.options
end
@@ -24,6 +30,7 @@ module Ci
end
expose :variables
+ expose :artifacts_file, using: ArtifactFile
end
class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 7e4986b6af3..02502333756 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,6 +1,8 @@
module Ci
module API
module Helpers
+ BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+ BUILD_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 60
def authenticate_runners!
@@ -15,6 +17,11 @@ module Ci
forbidden! unless project.valid_token?(params[:project_token])
end
+ def authenticate_build_token!(build)
+ token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+ forbidden! unless token && build.valid_token?(token)
+ end
+
def update_runner_last_contact
# Use a random threshold to prevent beating DB updates
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
@@ -32,6 +39,10 @@ module Ci
info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
current_runner.update(info)
end
+
+ def max_artifacts_size
+ current_application_settings.max_artifacts_size.megabytes.to_i
+ end
end
end
end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 0f57a4f53ab..6f9af5388ca 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -5,7 +5,7 @@ module Ci
DEFAULT_STAGES = %w(build test deploy)
DEFAULT_STAGE = 'test'
ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
- ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+ ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
attr_reader :before_script, :image, :services, :variables, :path
@@ -77,7 +77,8 @@ module Ci
when: job[:when] || 'on_success',
options: {
image: job[:image] || @image,
- services: job[:services] || @services
+ services: job[:services] || @services,
+ artifacts: job[:artifacts]
}.compact
}
end
@@ -159,6 +160,10 @@ module Ci
raise ValidationError, "#{name} job: except parameter should be an array of strings"
end
+ if job[:artifacts] && !validate_array_of_strings(job[:artifacts])
+ raise ValidationError, "#{name}: artifacts parameter should be an array of strings"
+ end
+
if job[:allow_failure] && !job[:allow_failure].in?([true, false])
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end
diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb
new file mode 100644
index 00000000000..4e3c6d3c773
--- /dev/null
+++ b/lib/file_streamer.rb
@@ -0,0 +1,16 @@
+class FileStreamer #:nodoc:
+ attr_reader :to_path
+
+ def initialize(path)
+ @to_path = path
+ end
+
+ # Stream the file's contents if Rack::Sendfile isn't present.
+ def each
+ File.open(to_path, 'rb') do |file|
+ while chunk = file.read(16384)
+ yield chunk
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index cd84afa31d5..2d3e32d9539 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -25,6 +25,7 @@ module Gitlab
session_expire_delay: Settings.gitlab['session_expire_delay'],
import_sources: Settings.gitlab['import_sources'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+ max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
)
end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e767027dc29..e511d5e4b4b 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -131,6 +131,22 @@ server {
return 418;
}
+ # Build artifacts should be submitted to this location
+ location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
+ # Build artifacts should be submitted to this location
+ location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 4d31e31f8d5..47b1ec8cb0c 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -178,6 +178,22 @@ server {
return 418;
}
+ # Build artifacts should be submitted to this location
+ location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
+ # Build artifacts should be submitted to this location
+ location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+ client_max_body_size 0;
+ # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+ error_page 418 = @gitlab-git-http-server;
+ return 418;
+ }
+
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index f20c7f71ba5..3c46bcea40e 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -12,6 +12,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:create"].invoke
Rake::Task["gitlab:backup:uploads:create"].invoke
Rake::Task["gitlab:backup:builds:create"].invoke
+ Rake::Task["gitlab:backup:artifacts:create"].invoke
backup = Backup::Manager.new
backup.pack
@@ -32,6 +33,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+ Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
Rake::Task["gitlab:shell:setup"].invoke
backup.cleanup
@@ -113,6 +115,25 @@ namespace :gitlab do
end
end
+ namespace :artifacts do
+ task create: :environment do
+ $progress.puts "Dumping artifacts ... ".blue
+
+ if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+ $progress.puts "[SKIPPED]".cyan
+ else
+ Backup::Artifacts.new.dump
+ $progress.puts "done".green
+ end
+ end
+
+ task restore: :environment do
+ $progress.puts "Restoring artifacts ... ".blue
+ Backup::Artifacts.new.restore
+ $progress.puts "done".green
+ end
+ end
+
def configure_cron_mode
if ENV['CRON']
# We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
new file mode 100644
index 00000000000..d4291f012d3
--- /dev/null
+++ b/lib/uploaded_file.rb
@@ -0,0 +1,37 @@
+require "tempfile"
+require "fileutils"
+
+# Taken from: Rack::Test::UploadedFile
+class UploadedFile
+
+ # The filename, *not* including the path, of the "uploaded" file
+ attr_reader :original_filename
+
+ # The tempfile
+ attr_reader :tempfile
+
+ # The content type of the "uploaded" file
+ attr_accessor :content_type
+
+ def initialize(path, filename, content_type = "text/plain")
+ raise "#{path} file does not exist" unless ::File.exist?(path)
+
+ @content_type = content_type
+ @original_filename = filename || ::File.basename(path)
+ @tempfile = File.new(path, 'rb')
+ end
+
+ def path
+ @tempfile.path
+ end
+
+ alias_method :local_path, :path
+
+ def method_missing(method_name, *args, &block) #:nodoc:
+ @tempfile.__send__(method_name, *args, &block)
+ end
+
+ def respond_to?(method_name, include_private = false) #:nodoc:
+ @tempfile.respond_to?(method_name, include_private) || super
+ end
+end
diff --git a/shared/artifacts/.gitkeep b/shared/artifacts/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/shared/artifacts/.gitkeep
diff --git a/shared/tmp/artifacts-cache/.gitkeep b/shared/tmp/artifacts-cache/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/shared/tmp/artifacts-cache/.gitkeep
diff --git a/shared/tmp/artifacts-uploads/.gitkeep b/shared/tmp/artifacts-uploads/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/shared/tmp/artifacts-uploads/.gitkeep
diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb
index 158e85e598f..5213ce1099f 100644
--- a/spec/features/builds_spec.rb
+++ b/spec/features/builds_spec.rb
@@ -1,6 +1,8 @@
require 'spec_helper'
describe "Builds" do
+ let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
+
before do
login_as(:user)
@commit = FactoryGirl.create :ci_commit
@@ -66,6 +68,15 @@ describe "Builds" do
it { expect(page).to have_content @commit.sha[0..7] }
it { expect(page).to have_content @commit.git_commit_message }
it { expect(page).to have_content @commit.git_author_name }
+
+ context "Download artifacts" do
+ before do
+ @build.update_attributes(artifacts_file: artifacts_file)
+ visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
+ end
+
+ it { expect(page).to have_content 'Download artifacts' }
+ end
end
describe "POST /:project/builds/:id/cancel" do
@@ -90,4 +101,14 @@ describe "Builds" do
it { expect(page).to have_content 'pending' }
it { expect(page).to have_content 'Cancel' }
end
+
+ describe "GET /:project/builds/:id/download" do
+ before do
+ @build.update_attributes(artifacts_file: artifacts_file)
+ visit namespace_project_build_path(@gl_project.namespace, @gl_project, @build)
+ click_link 'Download artifacts'
+ end
+
+ it { expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type) }
+ end
end
diff --git a/spec/features/commits_spec.rb b/spec/features/commits_spec.rb
index 340924fafe7..90739cd6a28 100644
--- a/spec/features/commits_spec.rb
+++ b/spec/features/commits_spec.rb
@@ -19,7 +19,7 @@ describe "Commits" do
stub_ci_commit_to_return_yaml_file
end
- describe "GET /:project/commits/:sha" do
+ describe "GET /:project/commits/:sha/ci" do
before do
visit ci_status_path(@commit)
end
@@ -29,6 +29,20 @@ describe "Commits" do
it { expect(page).to have_content @commit.git_author_name }
end
+ context "Download artifacts" do
+ let(:artifacts_file) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
+
+ before do
+ @build.update_attributes(artifacts_file: artifacts_file)
+ end
+
+ it do
+ visit ci_status_path(@commit)
+ click_on "Download artifacts"
+ expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type)
+ end
+ end
+
describe "Cancel all builds" do
it "cancels commit" do
visit ci_status_path(@commit)
diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
index 9963f76f993..5e3779af19e 100644
--- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
+++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb
@@ -333,6 +333,37 @@ module Ci
end
end
+ describe "Artifacts" do
+ it "returns artifacts when defined" do
+ config = YAML.dump({
+ image: "ruby:2.1",
+ services: ["mysql"],
+ before_script: ["pwd"],
+ rspec: { artifacts: ["logs/", "binaries/"], script: "rspec" }
+ })
+
+ config_processor = GitlabCiYamlProcessor.new(config)
+
+ expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(1)
+ expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({
+ except: nil,
+ stage: "test",
+ stage_idx: 1,
+ name: :rspec,
+ only: nil,
+ commands: "pwd\nrspec",
+ tag_list: [],
+ options: {
+ image: "ruby:2.1",
+ services: ["mysql"],
+ artifacts: ["logs/", "binaries/"]
+ },
+ when: "on_success",
+ allow_failure: false
+ })
+ end
+ end
+
describe "Error handling" do
it "indicates that object is invalid" do
expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError)
@@ -491,6 +522,13 @@ module Ci
GitlabCiYamlProcessor.new(config, path)
end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always")
end
+
+ it "returns errors if job artifacts is not an array of strings" do
+ config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: "string" } })
+ expect do
+ GitlabCiYamlProcessor.new(config)
+ end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts parameter should be an array of strings")
+ end
end
end
end
diff --git a/spec/models/build_spec.rb b/spec/models/build_spec.rb
index 7f5abb83ac2..839b4c6b16e 100644
--- a/spec/models/build_spec.rb
+++ b/spec/models/build_spec.rb
@@ -400,4 +400,19 @@ describe Ci::Build do
end
end
end
+
+ describe :download_url do
+ subject { build.download_url }
+
+ it "should be nil if artifact doesn't exist" do
+ build.update_attributes(artifacts_file: nil)
+ is_expected.to be_nil
+ end
+
+ it 'should be nil if artifact exist' do
+ gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif')
+ build.update_attributes(artifacts_file: gif)
+ is_expected.to_not be_nil
+ end
+ end
end
diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb
index d26a300ed82..a9ef2fe5885 100644
--- a/spec/requests/api/users_spec.rb
+++ b/spec/requests/api/users_spec.rb
@@ -343,8 +343,9 @@ describe API::API, api: true do
end.to change{ user.keys.count }.by(1)
end
- it "should raise error for invalid ID" do
- expect{post api("/users/ASDF/keys", admin) }.to raise_error(ActionController::RoutingError)
+ it "should return 405 for invalid ID" do
+ post api("/users/ASDF/keys", admin)
+ expect(response.status).to eq(405)
end
end
@@ -374,9 +375,9 @@ describe API::API, api: true do
expect(json_response.first['title']).to eq(key.title)
end
- it "should return 404 for invalid ID" do
+ it "should return 405 for invalid ID" do
get api("/users/ASDF/keys", admin)
- expect(response.status).to eq(404)
+ expect(response.status).to eq(405)
end
end
end
@@ -434,7 +435,8 @@ describe API::API, api: true do
end
it "should raise error for invalid ID" do
- expect{post api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError)
+ post api("/users/ASDF/emails", admin)
+ expect(response.status).to eq(405)
end
end
@@ -465,7 +467,8 @@ describe API::API, api: true do
end
it "should raise error for invalid ID" do
- expect{put api("/users/ASDF/emails", admin) }.to raise_error(ActionController::RoutingError)
+ put api("/users/ASDF/emails", admin)
+ expect(response.status).to eq(405)
end
end
end
diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb
index 88218a93e1f..92ea25a3723 100644
--- a/spec/requests/ci/api/builds_spec.rb
+++ b/spec/requests/ci/api/builds_spec.rb
@@ -41,7 +41,7 @@ describe Ci::API::API do
it "should return 404 error if no builds for specific runner" do
commit = FactoryGirl.create(:ci_commit, gl_project: shared_gl_project)
- FactoryGirl.create(:ci_build, commit: commit, status: 'pending' )
+ FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
post ci_api("/builds/register"), token: runner.token
@@ -50,7 +50,7 @@ describe Ci::API::API do
it "should return 404 error if no builds for shared runner" do
commit = FactoryGirl.create(:ci_commit, gl_project: gl_project)
- FactoryGirl.create(:ci_build, commit: commit, status: 'pending' )
+ FactoryGirl.create(:ci_build, commit: commit, status: 'pending')
post ci_api("/builds/register"), token: shared_runner.token
@@ -79,7 +79,7 @@ describe Ci::API::API do
{ "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
{ "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
{ "key" => "DB_NAME", "value" => "postgres", "public" => true },
- { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false },
+ { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }
])
end
@@ -122,5 +122,185 @@ describe Ci::API::API do
expect(build.reload.trace).to eq 'hello_world'
end
end
+
+ context "Artifacts" do
+ let(:file_upload) { fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') }
+ let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') }
+ let(:commit) { FactoryGirl.create(:ci_commit, gl_project: gl_project) }
+ let(:build) { FactoryGirl.create(:ci_build, commit: commit, runner_id: runner.id) }
+ let(:authorize_url) { ci_api("/builds/#{build.id}/artifacts/authorize") }
+ let(:post_url) { ci_api("/builds/#{build.id}/artifacts") }
+ let(:delete_url) { ci_api("/builds/#{build.id}/artifacts") }
+ let(:get_url) { ci_api("/builds/#{build.id}/artifacts") }
+ let(:headers) { { "Gitlab-Git-Http-Server" => "1.0" } }
+ let(:headers_with_token) { headers.merge(Ci::API::Helpers::BUILD_TOKEN_HEADER => build.project.token) }
+
+ describe "POST /builds/:id/artifacts/authorize" do
+ context "should authorize posting artifact to running build" do
+ before do
+ build.run!
+ end
+
+ it "using token as parameter" do
+ post authorize_url, { token: build.project.token }, headers
+ expect(response.status).to eq(200)
+ expect(json_response["temp_path"]).to_not be_nil
+ end
+
+ it "using token as header" do
+ post authorize_url, {}, headers_with_token
+ expect(response.status).to eq(200)
+ expect(json_response["temp_path"]).to_not be_nil
+ end
+ end
+
+ context "should fail to post too large artifact" do
+ before do
+ build.run!
+ end
+
+ it "using token as parameter" do
+ settings = Gitlab::CurrentSettings::current_application_settings
+ settings.update_attributes(max_artifacts_size: 0)
+ post authorize_url, { token: build.project.token, filesize: 100 }, headers
+ expect(response.status).to eq(413)
+ end
+
+ it "using token as header" do
+ settings = Gitlab::CurrentSettings::current_application_settings
+ settings.update_attributes(max_artifacts_size: 0)
+ post authorize_url, { filesize: 100 }, headers_with_token
+ expect(response.status).to eq(413)
+ end
+ end
+
+ context "should get denied" do
+ it do
+ post authorize_url, { token: 'invalid', filesize: 100 }
+ expect(response.status).to eq(403)
+ end
+ end
+ end
+
+ describe "POST /builds/:id/artifacts" do
+ context "Disable sanitizer" do
+ before do
+ # by configuring this path we allow to pass temp file from any path
+ allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return('/')
+ end
+
+ context "should post artifact to running build" do
+ before do
+ build.run!
+ end
+
+ it do
+ upload_artifacts(file_upload, headers_with_token)
+ expect(response.status).to eq(201)
+ expect(json_response["artifacts_file"]["filename"]).to eq(file_upload.original_filename)
+ end
+
+ it "updates artifact" do
+ upload_artifacts(file_upload, headers_with_token)
+ upload_artifacts(file_upload2, headers_with_token)
+ expect(response.status).to eq(201)
+ expect(json_response["artifacts_file"]["filename"]).to eq(file_upload2.original_filename)
+ end
+ end
+
+ context "should fail to post too large artifact" do
+ before do
+ build.run!
+ end
+
+ it do
+ settings = Gitlab::CurrentSettings::current_application_settings
+ settings.update_attributes(max_artifacts_size: 0)
+ upload_artifacts(file_upload, headers_with_token)
+ expect(response.status).to eq(413)
+ end
+ end
+
+ context "should fail to post artifacts without file" do
+ before do
+ build.run!
+ end
+
+ it do
+ post post_url, {}, headers_with_token
+ expect(response.status).to eq(400)
+ end
+ end
+
+ context "should fail to post artifacts without GitLab-Workhorse" do
+ before do
+ build.run!
+ end
+
+ it do
+ post post_url, { token: build.project.token }, {}
+ expect(response.status).to eq(403)
+ end
+ end
+ end
+
+ context "should fail to post artifacts for outside of tmp path" do
+ before do
+ # by configuring this path we allow to pass file from @tmpdir only
+ # but all temporary files are stored in system tmp directory
+ @tmpdir = Dir.mktmpdir
+ allow(ArtifactUploader).to receive(:artifacts_upload_path).and_return(@tmpdir)
+ build.run!
+ end
+
+ after do
+ FileUtils.remove_entry @tmpdir
+ end
+
+ it do
+ upload_artifacts(file_upload, headers_with_token)
+ expect(response.status).to eq(400)
+ end
+ end
+
+ def upload_artifacts(file, headers = {})
+ params = {
+ file: file.path,
+ filename: file.original_filename,
+ }
+ post post_url, params, headers
+ end
+ end
+
+ describe "DELETE /builds/:id/artifacts" do
+ before do
+ build.run!
+ post delete_url, token: build.project.token, file: file_upload
+ end
+
+ it "should delete artifact build" do
+ build.success
+ delete delete_url, token: build.project.token
+ expect(response.status).to eq(200)
+ end
+ end
+
+ describe "GET /builds/:id/artifacts" do
+ before do
+ build.run!
+ end
+
+ it "should download artifact" do
+ build.update_attributes(artifacts_file: file_upload)
+ get get_url, token: build.project.token
+ expect(response.status).to eq(200)
+ end
+
+ it "should fail to download if no artifact uploaded" do
+ get get_url, token: build.project.token
+ expect(response.status).to eq(404)
+ end
+ end
+ end
end
end
diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 386ac9c8372..06559c3925d 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -16,7 +16,7 @@ describe 'gitlab:app namespace rake task' do
end
def reenable_backup_sub_tasks
- %w{db repo uploads builds}.each do |subtask|
+ %w{db repo uploads builds artifacts}.each do |subtask|
Rake::Task["gitlab:backup:#{subtask}:create"].reenable
end
end
@@ -56,6 +56,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke)
+ expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive(:invoke)
expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke)
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end
@@ -113,19 +114,20 @@ describe 'gitlab:app namespace rake task' do
it 'should set correct permissions on the tar contents' do
tar_contents, exit_status = Gitlab::Popen.popen(
- %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
+ %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
)
expect(exit_status).to eq(0)
expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('repositories/')
expect(tar_contents).to match('builds.tar.gz')
- expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz)\/$/)
+ expect(tar_contents).to match('artifacts.tar.gz')
+ expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz)\/$/)
end
it 'should delete temp directories' do
temp_dirs = Dir.glob(
- File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds}')
+ File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts}')
)
expect(temp_dirs).to be_empty
@@ -161,12 +163,13 @@ describe 'gitlab:app namespace rake task' do
it "does not contain skipped item" do
tar_contents, _exit_status = Gitlab::Popen.popen(
- %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz}
+ %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz}
)
expect(tar_contents).to match('db/')
expect(tar_contents).to match('uploads.tar.gz')
expect(tar_contents).to match('builds.tar.gz')
+ expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).not_to match('repositories/')
end
@@ -178,6 +181,7 @@ describe 'gitlab:app namespace rake task' do
expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke
expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke
expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke
+ expect(Rake::Task["gitlab:backup:artifacts:restore"]).to receive :invoke
expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke
expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error
end