diff options
25 files changed, 705 insertions, 263 deletions
@@ -210,6 +210,9 @@ gem 'mousetrap-rails', '~> 1.4.6' # Detect and convert string character encoding gem 'charlock_holmes', '~> 0.7.3' +# Parse duration +gem 'chronic_duration', '~> 0.10.6' + gem "sass-rails", '~> 5.0.0' gem "coffee-rails", '~> 4.1.0' gem "uglifier", '~> 2.7.2' diff --git a/Gemfile.lock b/Gemfile.lock index dfc15700494..2b2e2d2bb07 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -133,6 +133,8 @@ GEM mime-types (>= 1.16) cause (0.1) charlock_holmes (0.7.3) + chronic_duration (0.10.6) + numerizer (~> 0.1.1) chunky_png (1.3.5) cliver (0.3.2) coderay (1.1.0) @@ -424,6 +426,7 @@ GEM nokogiri (1.6.8) mini_portile2 (~> 2.1.0) pkg-config (~> 1.1.7) + numerizer (0.1.1) oauth (0.4.7) oauth2 (1.0.0) faraday (>= 0.8, < 0.10) @@ -857,6 +860,7 @@ DEPENDENCIES capybara-screenshot (~> 1.0.0) carrierwave (~> 0.10.0) charlock_holmes (~> 0.7.3) + chronic_duration (~> 0.10.6) coffee-rails (~> 4.1.0) connection_pool (~> 2.0) coveralls (~> 0.8.2) diff --git a/app/assets/javascripts/ci/build.coffee b/app/assets/javascripts/ci/build.coffee index f763ba96e33..2d515d7efa2 100644 --- a/app/assets/javascripts/ci/build.coffee +++ b/app/assets/javascripts/ci/build.coffee @@ -17,6 +17,8 @@ class @CiBuild .off 'resize.build' .on 'resize.build', @hideSidebar + @updateArtifactRemoveDate() + if $('#build-trace').length @getInitialBuildTrace() @initScrollButtonAffix() @@ -103,3 +105,10 @@ class @CiBuild $('.js-build-sidebar') .removeClass 'right-sidebar-collapsed' .addClass 'right-sidebar-expanded' + + updateArtifactRemoveDate: -> + $date = $('.js-artifacts-remove') + + if $date.length + date = $date.text() + $date.text $.timefor(new Date(date), ' ') diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index 832d7deb57d..f11c8321464 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -1,22 +1,18 @@ class Projects::ArtifactsController < Projects::ApplicationController layout 'project' before_action :authorize_read_build! + before_action :authorize_update_build!, only: [:keep] + before_action :validate_artifacts! def download unless artifacts_file.file_storage? return redirect_to artifacts_file.url end - unless artifacts_file.exists? - return render_404 - end - send_file artifacts_file.path, disposition: 'attachment' end def browse - return render_404 unless build.artifacts? - directory = params[:path] ? "#{params[:path]}/" : '' @entry = build.artifacts_metadata_entry(directory) @@ -34,8 +30,17 @@ class Projects::ArtifactsController < Projects::ApplicationController end end + def keep + build.keep_artifacts! + redirect_to namespace_project_build_path(project.namespace, project, build) + end + private + def validate_artifacts! + render_404 unless build.artifacts? + end + def build @build ||= project.builds.find_by!(id: params[:build_id]) end diff --git a/app/helpers/time_helper.rb b/app/helpers/time_helper.rb index 8142f733e76..b04b0a5114c 100644 --- a/app/helpers/time_helper.rb +++ b/app/helpers/time_helper.rb @@ -20,7 +20,6 @@ module TimeHelper end end - def date_from_to(from, to) "#{from.to_s(:short)} - #{to.to_s(:short)}" end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 6a64ca451f7..89a1f8b3f57 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -11,6 +11,8 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } + scope :with_artifacts, ->() { where.not(artifacts_file: nil) } + scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } mount_uploader :artifacts_file, ArtifactUploader mount_uploader :artifacts_metadata, ArtifactUploader @@ -317,7 +319,7 @@ module Ci end def artifacts? - artifacts_file.exists? + !artifacts_expired? && artifacts_file.exists? end def artifacts_metadata? @@ -328,11 +330,15 @@ module Ci Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path, **options).to_entry end + def erase_artifacts! + remove_artifacts_file! + remove_artifacts_metadata! + end + def erase(opts = {}) return false unless erasable? - remove_artifacts_file! - remove_artifacts_metadata! + erase_artifacts! erase_trace! update_erased!(opts[:erased_by]) end @@ -345,6 +351,25 @@ module Ci !self.erased_at.nil? end + def artifacts_expired? + artifacts_expire_at && artifacts_expire_at < Time.now + end + + def artifacts_expire_in + artifacts_expire_at - Time.now if artifacts_expire_at + end + + def artifacts_expire_in=(value) + self.artifacts_expire_at = + if value + Time.now + ChronicDuration.parse(value) + end + end + + def keep_artifacts! + self.update(artifacts_expire_at: nil) + end + private def erase_trace! @@ -352,7 +377,7 @@ module Ci end def update_erased!(user = nil) - self.update(erased_by: user, erased_at: Time.now) + self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil) end def yaml_variables diff --git a/app/views/projects/builds/_sidebar.html.haml b/app/views/projects/builds/_sidebar.html.haml index 5d931389dfb..7127acf388b 100644 --- a/app/views/projects/builds/_sidebar.html.haml +++ b/app/views/projects/builds/_sidebar.html.haml @@ -11,19 +11,33 @@ %p.build-detail-row #{@build.coverage}% - - if can?(current_user, :read_build, @project) && @build.artifacts? + - if can?(current_user, :read_build, @project) && (@build.artifacts? || @build.artifacts_expired?) .block{ class: ("block-first" if !@build.coverage) } .title Build artifacts - .btn-group.btn-group-justified{ role: :group } - = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do - Download + - if @build.artifacts_expired? + %p.build-detail-row + The artifacts were removed + #{time_ago_with_tooltip(@build.artifacts_expire_at)} + - elsif @build.artifacts_expire_at + %p.build-detail-row + The artifacts will be removed in + %span.js-artifacts-remove= @build.artifacts_expire_at - - if @build.artifacts_metadata? - = link_to browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do - Browse + - if @build.artifacts? + .btn-group.btn-group-justified{ role: :group } + - if @build.artifacts_expire_at + = link_to keep_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do + Keep - .block{ class: ("block-first" if !@build.coverage && !(can?(current_user, :read_build, @project) && @build.artifacts?)) } + = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do + Download + + - if @build.artifacts_metadata? + = link_to browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do + Browse + + .block{ class: ("block-first" if !@build.coverage && !(can?(current_user, :read_build, @project) && (@build.artifacts? || @build.artifacts_expired?))) } .title Build details - if @build.retryable? diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb new file mode 100644 index 00000000000..c64ea108d52 --- /dev/null +++ b/app/workers/expire_build_artifacts_worker.rb @@ -0,0 +1,13 @@ +class ExpireBuildArtifactsWorker + include Sidekiq::Worker + + def perform + Rails.logger.info 'Cleaning old build artifacts' + + builds = Ci::Build.with_expired_artifacts + builds.find_each(batch_size: 50).each do |build| + Rails.logger.debug "Removing artifacts build #{build.id}..." + build.erase_artifacts! + end + end +end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 1048ef6e243..75e1a3c1093 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -164,6 +164,9 @@ production: &base # Flag stuck CI builds as failed stuck_ci_builds_worker: cron: "0 0 * * *" + # Remove expired build artifacts + expire_build_artifacts_worker: + cron: "50 * * * *" # Periodically run 'git fsck' on all repositories. If started more than # once per hour you will have concurrent 'git fsck' jobs. repository_check_worker: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 436751b9d16..916fd33e767 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -279,6 +279,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker' +Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *' +Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker' Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *' Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::BatchWorker' diff --git a/config/initializers/chronic_duration.rb b/config/initializers/chronic_duration.rb new file mode 100644 index 00000000000..b65b06c813a --- /dev/null +++ b/config/initializers/chronic_duration.rb @@ -0,0 +1 @@ +ChronicDuration.raise_exceptions = true diff --git a/config/routes.rb b/config/routes.rb index 95fbe7dd9df..59724b737f6 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -722,6 +722,7 @@ Rails.application.routes.draw do get :download get :browse, path: 'browse(/*path)', format: false get :file, path: 'file/*path', format: false + post :keep end end diff --git a/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb b/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb new file mode 100644 index 00000000000..915167b038d --- /dev/null +++ b/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb @@ -0,0 +1,5 @@ +class AddArtifactsExpireDateToCiBuilds < ActiveRecord::Migration + def change + add_column :ci_builds, :artifacts_expire_at, :timestamp + end +end diff --git a/doc/api/builds.md b/doc/api/builds.md index 5669bd0cdda..de998944352 100644 --- a/doc/api/builds.md +++ b/doc/api/builds.md @@ -21,85 +21,85 @@ Example of response ```json [ - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.802Z", - "artifacts_file": { - "filename": "artifacts.zip", - "size": 1000 - }, - "finished_at": "2015-12-24T17:54:27.895Z", - "id": 7, - "name": "teaspoon", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:27.722Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.802Z", + "artifacts_file": { + "filename": "artifacts.zip", + "size": 1000 }, - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.727Z", - "artifacts_file": null, - "finished_at": "2015-12-24T17:54:24.921Z", - "id": 6, - "name": "spinach:other", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:24.729Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + "finished_at": "2015-12-24T17:54:27.895Z", + "id": 7, + "name": "teaspoon", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:27.722Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" + } + }, + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.727Z", + "artifacts_file": null, + "finished_at": "2015-12-24T17:54:24.921Z", + "id": 6, + "name": "spinach:other", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:24.729Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" } + } ] ``` @@ -125,68 +125,68 @@ Example of response ```json [ - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2016-01-11T10:13:33.506Z", - "artifacts_file": null, - "finished_at": "2016-01-11T10:14:09.526Z", - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": null, - "status": "canceled", - "tag": false, - "user": null + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." }, - { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.957Z", - "artifacts_file": null, - "finished_at": "2015-12-24T17:54:33.913Z", - "id": 9, - "name": "brakeman", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:33.727Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + "coverage": null, + "created_at": "2016-01-11T10:13:33.506Z", + "artifacts_file": null, + "finished_at": "2016-01-11T10:14:09.526Z", + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": null, + "status": "canceled", + "tag": false, + "user": null + }, + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.957Z", + "artifacts_file": null, + "finished_at": "2015-12-24T17:54:33.913Z", + "id": 9, + "name": "brakeman", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:33.727Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" } + } ] ``` @@ -211,42 +211,42 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2015-12-24T15:51:21.880Z", - "artifacts_file": null, - "finished_at": "2015-12-24T17:54:31.198Z", - "id": 8, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": "2015-12-24T17:54:30.733Z", - "status": "failed", - "tag": false, - "user": { - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "bio": null, - "created_at": "2015-12-21T13:14:24.077Z", - "id": 1, - "is_admin": true, - "linkedin": "", - "name": "Administrator", - "skype": "", - "state": "active", - "twitter": "", - "username": "root", - "web_url": "http://gitlab.dev/u/root", - "website_url": "" - } + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2015-12-24T15:51:21.880Z", + "artifacts_file": null, + "finished_at": "2015-12-24T17:54:31.198Z", + "id": 8, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": "2015-12-24T17:54:30.733Z", + "status": "failed", + "tag": false, + "user": { + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "bio": null, + "created_at": "2015-12-21T13:14:24.077Z", + "id": 1, + "is_admin": true, + "linkedin": "", + "name": "Administrator", + "skype": "", + "state": "active", + "twitter": "", + "username": "root", + "web_url": "http://gitlab.dev/u/root", + "website_url": "" + } } ``` @@ -323,28 +323,28 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2016-01-11T10:13:33.506Z", - "artifacts_file": null, - "finished_at": "2016-01-11T10:14:09.526Z", - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": null, - "status": "canceled", - "tag": false, - "user": null + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2016-01-11T10:13:33.506Z", + "artifacts_file": null, + "finished_at": "2016-01-11T10:14:09.526Z", + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": null, + "status": "canceled", + "tag": false, + "user": null } ``` @@ -369,28 +369,28 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "created_at": "2016-01-11T10:13:33.506Z", - "artifacts_file": null, - "finished_at": null, - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "started_at": null, - "status": "pending", - "tag": false, - "user": null + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "created_at": "2016-01-11T10:13:33.506Z", + "artifacts_file": null, + "finished_at": null, + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "started_at": null, + "status": "pending", + "tag": false, + "user": null } ``` @@ -419,27 +419,77 @@ Example of response ```json { - "commit": { - "author_email": "admin@example.com", - "author_name": "Administrator", - "created_at": "2015-12-24T16:51:14.000+01:00", - "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", - "message": "Test the CI integration.", - "short_id": "0ff3ae19", - "title": "Test the CI integration." - }, - "coverage": null, - "download_url": null, - "id": 69, - "name": "rubocop", - "ref": "master", - "runner": null, - "stage": "test", - "created_at": "2016-01-11T10:13:33.506Z", - "started_at": "2016-01-11T10:13:33.506Z", - "finished_at": "2016-01-11T10:15:10.506Z", - "status": "failed", - "tag": false, - "user": null + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "download_url": null, + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "created_at": "2016-01-11T10:13:33.506Z", + "started_at": "2016-01-11T10:13:33.506Z", + "finished_at": "2016-01-11T10:15:10.506Z", + "status": "failed", + "tag": false, + "user": null +} +``` + +## Keep artifacts + +Prevents artifacts from being deleted when expiration is set. + +``` +POST /projects/:id/builds/:build_id/artifacts/keep +``` + +Parameters + +| Attribute | Type | required | Description | +|-------------|---------|----------|---------------------| +| `id` | integer | yes | The ID of a project | +| `build_id` | integer | yes | The ID of a build | + +Example request: + +``` +curl -X POST -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v3/projects/1/builds/1/artifacts/keep" +``` + +Example response: + +```json +{ + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "download_url": null, + "id": 69, + "name": "rubocop", + "ref": "master", + "runner": null, + "stage": "test", + "created_at": "2016-01-11T10:13:33.506Z", + "started_at": "2016-01-11T10:13:33.506Z", + "finished_at": "2016-01-11T10:15:10.506Z", + "status": "failed", + "tag": false, + "user": null } ``` diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 0707555e393..d71ce6d6b13 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -31,6 +31,7 @@ If you want a quick introduction to GitLab CI, follow our - [artifacts](#artifacts) - [artifacts:name](#artifacts-name) - [artifacts:when](#artifacts-when) + - [artifacts:expire_in](#artifacts-expire_in) - [dependencies](#dependencies) - [before_script and after_script](#before_script-and-after_script) - [Hidden jobs](#hidden-jobs) @@ -678,6 +679,40 @@ job: when: on_failure ``` +#### artifacts:expire_in + +>**Note:** +Introduced in GitLab 8.9 and GitLab Runner v1.3.0. + +`artifacts:expire_in` is used to remove uploaded artifacts after specified time. +By default artifacts are stored on GitLab forver. +`expire_in` allows to specify after what time the artifacts should be removed. +The artifacts will expire counting from the moment when they are uploaded and stored on GitLab. + +After artifacts uploading you can use the **Keep** button on build page to keep the artifacts forever. + +Artifacts are removed every hour, but they are not accessible after expire date. + +The value of `expire_in` is a elapsed time. The example of parsable values: +- '3 mins 4 sec' +- '2 hrs 20 min' +- '2h20min' +- '6 mos 1 day' +- '47 yrs 6 mos and 4d' +- '3 weeks and 2 days' + +--- + +**Example configurations** + +To expire artifacts after 1 week from the moment that they are uploaded: + +```yaml +job: + artifacts: + expire_in: 1 week +``` + ### dependencies >**Note:** diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 0ff8fa74a84..644e5a2a99d 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -166,6 +166,26 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end + + # Keep the artifacts to prevent them from being deleted + # + # Parameters: + # id (required) - the id of a project + # build_id (required) - The ID of a build + # Example Request: + # POST /projects/:id/builds/:build_id/artifacts/keep + post ':id/builds/:build_id/artifacts/keep' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build && build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 607359769d1..9f270f7b387 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -114,6 +114,7 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file + # expire_in (optional) - Specify when artifacts should expire (ex. 7d) # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition @@ -145,6 +146,7 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata + build.artifacts_expire_in = params['expire_in'] if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index a902ced35d7..3f5bdaba3f5 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } + expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end class BuildDetails < Build @@ -29,6 +29,7 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token + expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } expose :options do |model| model.options diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 40a5d180fd0..76d84433cbe 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -9,7 +9,7 @@ module Ci :allow_failure, :type, :stage, :when, :artifacts, :cache, :dependencies, :before_script, :after_script, :variables] ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] - ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when] + ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] attr_reader :before_script, :after_script, :image, :services, :path, :cache @@ -282,6 +282,10 @@ module Ci if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" end + + if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) + raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" + end end def validate_job_dependencies!(name, job) @@ -300,6 +304,12 @@ module Ci end end + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + def validate_array_of_strings(values) values.is_a?(Array) && values.all? { |value| validate_string(value) } end diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb index b8ecc356b4d..0fd95295388 100644 --- a/spec/features/builds_spec.rb +++ b/spec/features/builds_spec.rb @@ -97,6 +97,48 @@ describe "Builds" do end end + context 'Artifacts expire date' do + before do + @build.update_attributes(artifacts_file: artifacts_file, artifacts_expire_at: expire_at) + visit namespace_project_build_path(@project.namespace, @project, @build) + end + + context 'no expire date defined' do + let(:expire_at) { nil } + + it 'does not have the Keep button' do + page.within('.artifacts') do + expect(page).not_to have_content 'Keep' + end + end + end + + context 'when expire date is defined' do + let(:expire_at) { Time.now + 7.days } + + it 'keeps artifacts when Keep button is clicked' do + page.within('.artifacts') do + expect(page).to have_content 'The artifacts will be removed' + click_link 'Keep' + end + + expect(page).not_to have_link 'Keep' + expect(page).not_to have_content 'The artifacts will be removed' + end + end + + context 'when artifacts expired' do + let(:expire_at) { Time.now - 7.days } + + it 'does not have the Keep button' do + page.within('.artifacts') do + expect(page).to have_content 'The artifacts were removed' + expect(page).not_to have_link 'Keep' + end + end + end + end + context 'Build raw trace' do before do @build.run! diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index 304290d6608..5e1d2b8e4f5 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -573,7 +573,12 @@ module Ci services: ["mysql"], before_script: ["pwd"], rspec: { - artifacts: { paths: ["logs/", "binaries/"], untracked: true, name: "custom_name" }, + artifacts: { + paths: ["logs/", "binaries/"], + untracked: true, + name: "custom_name", + expire_in: "7d" + }, script: "rspec" } }) @@ -595,7 +600,8 @@ module Ci artifacts: { name: "custom_name", paths: ["logs/", "binaries/"], - untracked: true + untracked: true, + expire_in: "7d" } }, when: "on_success", @@ -992,6 +998,20 @@ EOT end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:when parameter should be on_success, on_failure or always") end + it "returns errors if job artifacts:expire_in is not an a string" do + config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { expire_in: 1 } } }) + expect do + GitlabCiYamlProcessor.new(config) + end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:expire_in parameter should be a duration") + end + + it "returns errors if job artifacts:expire_in is not an a valid duration" do + config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { expire_in: "7 elephants" } } }) + expect do + GitlabCiYamlProcessor.new(config) + end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:expire_in parameter should be a duration") + end + it "returns errors if job artifacts:untracked is not an array of strings" do config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { untracked: "string" } } }) expect do diff --git a/spec/models/build_spec.rb b/spec/models/build_spec.rb index 2beb6cc598d..c07832a4b5f 100644 --- a/spec/models/build_spec.rb +++ b/spec/models/build_spec.rb @@ -397,9 +397,32 @@ describe Ci::Build, models: true do context 'artifacts archive exists' do let(:build) { create(:ci_build, :artifacts) } it { is_expected.to be_truthy } + + context 'is expired' do + before { build.update(artifacts_expire_at: Time.now - 7.days) } + it { is_expected.to be_falsy } + end + + context 'is not expired' do + before { build.update(artifacts_expire_at: Time.now + 7.days) } + it { is_expected.to be_truthy } + end end end + describe '#artifacts_expired?' do + subject { build.artifacts_expired? } + + context 'is expired' do + before { build.update(artifacts_expire_at: Time.now - 7.days) } + it { is_expected.to be_falsy } + end + + context 'is not expired' do + before { build.update(artifacts_expire_at: Time.now + 7.days) } + it { is_expected.to be_truthy } + end + end describe '#artifacts_metadata?' do subject { build.artifacts_metadata? } @@ -412,7 +435,6 @@ describe Ci::Build, models: true do it { is_expected.to be_truthy } end end - describe '#repo_url' do let(:build) { create(:ci_build) } let(:project) { build.project } @@ -427,6 +449,50 @@ describe Ci::Build, models: true do it { is_expected.to include(project.web_url[7..-1]) } end + describe '#artifacts_expire_in' do + subject { build.artifacts_expire_in } + it { is_expected.to be_nil } + + context 'when artifacts_expire_at is specified' do + let(:expire_at) { Time.now + 7.days } + + before { build.artifacts_expire_at = expire_at } + + it { is_expected.to be_within(5).of(expire_at - Time.now) } + end + end + + describe '#artifacts_expire_in=' do + subject { build.artifacts_expire_in } + + it 'when assigning valid duration' do + build.artifacts_expire_in = '7 days' + + is_expected.to be_within(10).of(7.days.to_i) + end + + it 'when assigning invalid duration' do + expect { build.artifacts_expire_in = '7 elephants' }.not_to raise_error + is_expected.to be_nil + end + + it 'when resseting value' do + build.artifacts_expire_in = nil + + is_expected.to be_nil + end + end + + describe '#keep_artifacts!' do + let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) } + + it 'to reset expire_at' do + build.keep_artifacts! + + expect(build.artifacts_expire_at).to be_nil + end + end + describe '#depends_on_builds' do let!(:build) { create(:ci_build, pipeline: pipeline, name: 'build', stage_idx: 0, stage: 'build') } let!(:rspec_test) { create(:ci_build, pipeline: pipeline, name: 'rspec', stage_idx: 1, stage: 'test') } diff --git a/spec/requests/api/builds_spec.rb b/spec/requests/api/builds_spec.rb index 6cb7be188ef..ac85f340922 100644 --- a/spec/requests/api/builds_spec.rb +++ b/spec/requests/api/builds_spec.rb @@ -241,4 +241,30 @@ describe API::API, api: true do end end end + + describe 'POST /projects/:id/builds/:build_id/artifacts/keep' do + before do + post api("/projects/#{project.id}/builds/#{build.id}/artifacts/keep", user) + end + + context 'artifacts did not expire' do + let(:build) do + create(:ci_build, :trace, :artifacts, :success, + project: project, pipeline: pipeline, artifacts_expire_at: Time.now + 7.days) + end + + it 'keeps artifacts' do + expect(response.status).to eq 200 + expect(build.reload.artifacts_expire_at).to be_nil + end + end + + context 'no artifacts' do + let(:build) { create(:ci_build, project: project, pipeline: pipeline) } + + it 'responds with not found' do + expect(response.status).to eq 404 + end + end + end end diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb index e8508f8f950..7e50bea90d1 100644 --- a/spec/requests/ci/api/builds_spec.rb +++ b/spec/requests/ci/api/builds_spec.rb @@ -364,6 +364,42 @@ describe Ci::API::API do end end + context 'with an expire date' do + let!(:artifacts) { file_upload } + + let(:post_data) do + { 'file.path' => artifacts.path, + 'file.name' => artifacts.original_filename, + 'expire_in' => expire_in } + end + + before do + post(post_url, post_data, headers_with_token) + end + + context 'with an expire_in given' do + let(:expire_in) { '7 days' } + + it 'updates when specified' do + build.reload + expect(response.status).to eq(201) + expect(json_response['artifacts_expire_at']).not_to be_empty + expect(build.artifacts_expire_at).to be_within(5.minutes).of(Time.now + 7.days) + end + end + + context 'with no expire_in given' do + let(:expire_in) { nil } + + it 'ignores if not specified' do + build.reload + expect(response.status).to eq(201) + expect(json_response['artifacts_expire_at']).to be_nil + expect(build.artifacts_expire_at).to be_nil + end + end + end + context "artifacts file is too large" do it "should fail to post too large artifact" do stub_application_setting(max_artifacts_size: 0) diff --git a/spec/workers/expire_build_artifacts_worker_spec.rb b/spec/workers/expire_build_artifacts_worker_spec.rb new file mode 100644 index 00000000000..eb8afb20275 --- /dev/null +++ b/spec/workers/expire_build_artifacts_worker_spec.rb @@ -0,0 +1,49 @@ +require 'spec_helper' + +describe ExpireBuildArtifactsWorker do + include RepoHelpers + + let(:worker) { described_class.new } + + describe '#perform' do + before { build } + + subject! { worker.perform } + + context 'with expired artifacts' do + let!(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) } + + it 'does expire' do + expect(build.reload.artifacts_expired?).to be_truthy + end + end + + context 'with not yet expired artifacts' do + let!(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) } + + it 'does not expire' do + expect(build.reload.artifacts_expired?).to be_truthy + end + end + + context 'without expire date' do + let!(:build) { create(:ci_build, :artifacts) } + + it 'does not expire' do + expect(build.reload.artifacts_expired?).to be_falsey + end + end + + context 'for expired artifacts' do + let!(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) } + + it 'does not erase artifacts' do + expect_any_instance_of(Ci::Build).not_to have_received(:erase_artifacts!) + end + + it 'does expire' do + expect(build.reload.artifacts_expired?).to be_truthy + end + end + end +end |