diff options
Diffstat (limited to 'app/models/ci')
| -rw-r--r-- | app/models/ci/build.rb | 178 | ||||
| -rw-r--r-- | app/models/ci/commit.rb | 215 | ||||
| -rw-r--r-- | app/models/ci/pipeline.rb | 203 | ||||
| -rw-r--r-- | app/models/ci/runner.rb | 39 | ||||
| -rw-r--r-- | app/models/ci/runner_project.rb | 12 | ||||
| -rw-r--r-- | app/models/ci/trigger.rb | 13 | ||||
| -rw-r--r-- | app/models/ci/trigger_request.rb | 14 | ||||
| -rw-r--r-- | app/models/ci/variable.rb | 19 |
8 files changed, 333 insertions, 360 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 7d33838044b..d618c84e983 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -1,44 +1,5 @@ -# == Schema Information -# -# Table name: ci_builds -# -# id :integer not null, primary key -# project_id :integer -# status :string(255) -# finished_at :datetime -# trace :text -# created_at :datetime -# updated_at :datetime -# started_at :datetime -# runner_id :integer -# coverage :float -# commit_id :integer -# commands :text -# job_id :integer -# name :string(255) -# deploy :boolean default(FALSE) -# options :text -# allow_failure :boolean default(FALSE), not null -# stage :string(255) -# trigger_request_id :integer -# stage_idx :integer -# tag :boolean -# ref :string(255) -# user_id :integer -# type :string(255) -# target_url :string(255) -# description :string(255) -# artifacts_file :text -# gl_project_id :integer -# artifacts_metadata :text -# erased_by_id :integer -# erased_at :datetime -# - module Ci class Build < CommitStatus - LAZY_ATTRIBUTES = ['trace'] - belongs_to :runner, class_name: 'Ci::Runner' belongs_to :trigger_request, class_name: 'Ci::TriggerRequest' belongs_to :erased_by, class_name: 'User' @@ -50,25 +11,19 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } - scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) } + scope :with_artifacts, ->() { where.not(artifacts_file: nil) } + scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } mount_uploader :artifacts_file, ArtifactUploader mount_uploader :artifacts_metadata, ArtifactUploader acts_as_taggable - # To prevent db load megabytes of data from trace - default_scope -> { select(Ci::Build.columns_without_lazy) } - before_destroy { project } - class << self - def columns_without_lazy - (column_names - LAZY_ATTRIBUTES).map do |column_name| - "#{table_name}.#{column_name}" - end - end + after_create :execute_hooks + class << self def last_month where('created_at > ?', Date.today - 1.month) end @@ -85,21 +40,23 @@ module Ci new_build.save end - def retry(build) + def retry(build, user = nil) new_build = Ci::Build.new(status: 'pending') new_build.ref = build.ref new_build.tag = build.tag new_build.options = build.options new_build.commands = build.commands new_build.tag_list = build.tag_list - new_build.gl_project_id = build.gl_project_id - new_build.commit_id = build.commit_id + new_build.project = build.project + new_build.pipeline = build.pipeline new_build.name = build.name new_build.allow_failure = build.allow_failure new_build.stage = build.stage new_build.stage_idx = build.stage_idx new_build.trigger_request = build.trigger_request + new_build.user = user new_build.save + MergeRequests::AddTodoWhenBuildFailsService.new(build.project, nil).close(new_build) new_build end end @@ -112,13 +69,24 @@ module Ci # We use around_transition to create builds for next stage as soon as possible, before the `after_*` is executed around_transition any => [:success, :failed, :canceled] do |build, block| block.call - build.commit.create_next_builds(build) if build.commit + build.pipeline.create_next_builds(build) if build.pipeline end after_transition any => [:success, :failed, :canceled] do |build| build.update_coverage build.execute_hooks end + + after_transition any => [:success] do |build| + if build.environment.present? + service = CreateDeploymentService.new(build.project, build.user, + environment: build.environment, + sha: build.sha, + ref: build.ref, + tag: build.tag) + service.execute(build) + end + end end def retryable? @@ -126,20 +94,24 @@ module Ci end def retried? - !self.commit.latest_statuses_for_ref(self.ref).include?(self) + !self.pipeline.statuses.latest.include?(self) end def depends_on_builds # Get builds of the same type - latest_builds = self.commit.builds.similar(self).latest + latest_builds = self.pipeline.builds.latest # Return builds from previous stages latest_builds.where('stage_idx < ?', stage_idx) end def trace_html - html = Ci::Ansi2html::convert(trace) if trace.present? - html || '' + trace_with_state[:html] || '' + end + + def trace_with_state(state = nil) + trace_with_state = Ci::Ansi2html::convert(trace, state) if trace.present? + trace_with_state || {} end def timeout @@ -152,16 +124,16 @@ module Ci def merge_request merge_requests = MergeRequest.includes(:merge_request_diff) - .where(source_branch: ref, source_project_id: commit.gl_project_id) + .where(source_branch: ref, source_project_id: pipeline.gl_project_id) .reorder(iid: :asc) merge_requests.find do |merge_request| - merge_request.commits.any? { |ci| ci.id == commit.sha } + merge_request.commits.any? { |ci| ci.id == pipeline.sha } end end def project_id - commit.project.id + pipeline.project_id end def project_name @@ -230,12 +202,33 @@ module Ci end end + def trace_length + if raw_trace + raw_trace.bytesize + else + 0 + end + end + def trace=(trace) - unless Dir.exists?(dir_to_trace) + recreate_trace_dir + File.write(path_to_trace, trace) + end + + def recreate_trace_dir + unless Dir.exist?(dir_to_trace) FileUtils.mkdir_p(dir_to_trace) end + end + private :recreate_trace_dir - File.write(path_to_trace, trace) + def append_trace(trace_part, offset) + recreate_trace_dir + + File.truncate(path_to_trace, offset) if File.exist?(path_to_trace) + File.open(path_to_trace, 'ab') do |f| + f.write(trace_part) + end end def dir_to_trace @@ -303,14 +296,20 @@ module Ci project.runners_token end - def valid_token? token + def valid_token?(token) project.valid_runners_token? token end def can_be_served?(runner) + return false unless has_tags? || runner.run_untagged? + (tag_list - runner.tag_list).empty? end + def has_tags? + tag_list.any? + end + def any_runners_online? project.any_runners? { |runner| runner.active? && runner.online? && can_be_served?(runner) } end @@ -324,10 +323,11 @@ module Ci build_data = Gitlab::BuildDataBuilder.build(self) project.execute_hooks(build_data.dup, :build_hooks) project.execute_services(build_data.dup, :build_hooks) + project.running_or_pending_build_count(force: true) end def artifacts? - artifacts_file.exists? + !artifacts_expired? && artifacts_file.exists? end def artifacts_metadata? @@ -338,11 +338,16 @@ module Ci Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path, **options).to_entry end + def erase_artifacts! + remove_artifacts_file! + remove_artifacts_metadata! + save + end + def erase(opts = {}) return false unless erasable? - remove_artifacts_file! - remove_artifacts_metadata! + erase_artifacts! erase_trace! update_erased!(opts[:erased_by]) end @@ -355,6 +360,25 @@ module Ci !self.erased_at.nil? end + def artifacts_expired? + artifacts_expire_at && artifacts_expire_at < Time.now + end + + def artifacts_expire_in + artifacts_expire_at - Time.now if artifacts_expire_at + end + + def artifacts_expire_in=(value) + self.artifacts_expire_at = + if value + Time.now + ChronicDuration.parse(value) + end + end + + def keep_artifacts! + self.update(artifacts_expire_at: nil) + end + private def erase_trace! @@ -362,14 +386,26 @@ module Ci end def update_erased!(user = nil) - self.update(erased_by: user, erased_at: Time.now) + self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil) end - private - def yaml_variables - if commit.config_processor - commit.config_processor.variables.map do |key, value| + global_yaml_variables + job_yaml_variables + end + + def global_yaml_variables + if pipeline.config_processor + pipeline.config_processor.global_variables.map do |key, value| + { key: key, value: value, public: true } + end + else + [] + end + end + + def job_yaml_variables + if pipeline.config_processor + pipeline.config_processor.job_variables(name).map do |key, value| { key: key, value: value, public: true } end else diff --git a/app/models/ci/commit.rb b/app/models/ci/commit.rb deleted file mode 100644 index f4cf7034b14..00000000000 --- a/app/models/ci/commit.rb +++ /dev/null @@ -1,215 +0,0 @@ -# == Schema Information -# -# Table name: ci_commits -# -# id :integer not null, primary key -# project_id :integer -# ref :string(255) -# sha :string(255) -# before_sha :string(255) -# push_data :text -# created_at :datetime -# updated_at :datetime -# tag :boolean default(FALSE) -# yaml_errors :text -# committed_at :datetime -# gl_project_id :integer -# - -module Ci - class Commit < ActiveRecord::Base - extend Ci::Model - - belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id - has_many :statuses, class_name: 'CommitStatus' - has_many :builds, class_name: 'Ci::Build' - has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest' - - validates_presence_of :sha - validate :valid_commit_sha - - def self.truncate_sha(sha) - sha[0...8] - end - - def to_param - sha - end - - def project_id - project.id - end - - def valid_commit_sha - if self.sha == Gitlab::Git::BLANK_SHA - self.errors.add(:sha, " cant be 00000000 (branch removal)") - end - end - - def git_author_name - commit_data.author_name if commit_data - end - - def git_author_email - commit_data.author_email if commit_data - end - - def git_commit_message - commit_data.message if commit_data - end - - def short_sha - Ci::Commit.truncate_sha(sha) - end - - def commit_data - @commit ||= project.commit(sha) - rescue - nil - end - - def stage - running_or_pending = statuses.latest.running_or_pending.ordered - running_or_pending.first.try(:stage) - end - - def create_builds(ref, tag, user, trigger_request = nil) - return unless config_processor - config_processor.stages.any? do |stage| - CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request, 'success').present? - end - end - - def create_next_builds(build) - return unless config_processor - - # don't create other builds if this one is retried - latest_builds = builds.similar(build).latest - return unless latest_builds.exists?(build.id) - - # get list of stages after this build - next_stages = config_processor.stages.drop_while { |stage| stage != build.stage } - next_stages.delete(build.stage) - - # get status for all prior builds - prior_builds = latest_builds.reject { |other_build| next_stages.include?(other_build.stage) } - status = Ci::Status.get_status(prior_builds) - - # create builds for next stages based - next_stages.any? do |stage| - CreateBuildsService.new.execute(self, stage, build.ref, build.tag, build.user, build.trigger_request, status).present? - end - end - - def refs - statuses.order(:ref).pluck(:ref).uniq - end - - def latest_statuses - @latest_statuses ||= statuses.latest.to_a - end - - def latest_statuses_for_ref(ref) - latest_statuses.select { |status| status.ref == ref } - end - - def matrix_builds(build = nil) - matrix_builds = builds.latest.ordered - matrix_builds = matrix_builds.similar(build) if build - matrix_builds.to_a - end - - def retried - @retried ||= (statuses.order(id: :desc) - statuses.latest) - end - - def status - if yaml_errors.present? - return 'failed' - end - - @status ||= Ci::Status.get_status(latest_statuses) - end - - def pending? - status == 'pending' - end - - def running? - status == 'running' - end - - def success? - status == 'success' - end - - def failed? - status == 'failed' - end - - def canceled? - status == 'canceled' - end - - def active? - running? || pending? - end - - def complete? - canceled? || success? || failed? - end - - def duration - duration_array = statuses.map(&:duration).compact - duration_array.reduce(:+).to_i - end - - def started_at - @started_at ||= statuses.order('started_at ASC').first.try(:started_at) - end - - def finished_at - @finished_at ||= statuses.order('finished_at DESC').first.try(:finished_at) - end - - def coverage - coverage_array = latest_statuses.map(&:coverage).compact - if coverage_array.size >= 1 - '%.2f' % (coverage_array.reduce(:+) / coverage_array.size) - end - end - - def config_processor - return nil unless ci_yaml_file - @config_processor ||= Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.path_with_namespace) - rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e - save_yaml_error(e.message) - nil - rescue - save_yaml_error("Undefined error") - nil - end - - def ci_yaml_file - @ci_yaml_file ||= begin - blob = project.repository.blob_at(sha, '.gitlab-ci.yml') - blob.load_all_data!(project.repository) - blob.data - end - rescue - nil - end - - def skip_ci? - git_commit_message =~ /(\[ci skip\])/ if git_commit_message - end - - private - - def save_yaml_error(error) - return if self.yaml_errors? - self.yaml_errors = error - save - end - end -end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb new file mode 100644 index 00000000000..5b264ecffc5 --- /dev/null +++ b/app/models/ci/pipeline.rb @@ -0,0 +1,203 @@ +module Ci + class Pipeline < ActiveRecord::Base + extend Ci::Model + include Statuseable + + self.table_name = 'ci_commits' + + belongs_to :project, class_name: '::Project', foreign_key: :gl_project_id + has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id + has_many :builds, class_name: 'Ci::Build', foreign_key: :commit_id + has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest', foreign_key: :commit_id + + validates_presence_of :sha + validates_presence_of :status + validate :valid_commit_sha + + # Invalidate object and save if when touched + after_touch :update_state + + def self.truncate_sha(sha) + sha[0...8] + end + + def self.stages + # We use pluck here due to problems with MySQL which doesn't allow LIMIT/OFFSET in queries + CommitStatus.where(pipeline: pluck(:id)).stages + end + + def project_id + project.id + end + + def valid_commit_sha + if self.sha == Gitlab::Git::BLANK_SHA + self.errors.add(:sha, " cant be 00000000 (branch removal)") + end + end + + def git_author_name + commit_data.author_name if commit_data + end + + def git_author_email + commit_data.author_email if commit_data + end + + def git_commit_message + commit_data.message if commit_data + end + + def short_sha + Ci::Pipeline.truncate_sha(sha) + end + + def commit_data + @commit ||= project.commit(sha) + rescue + nil + end + + def branch? + !tag? + end + + def retryable? + builds.latest.any? do |build| + build.failed? && build.retryable? + end + end + + def cancelable? + builds.running_or_pending.any? + end + + def cancel_running + builds.running_or_pending.each(&:cancel) + end + + def retry_failed(user) + builds.latest.failed.select(&:retryable?).each do |build| + Ci::Build.retry(build, user) + end + end + + def latest? + return false unless ref + commit = project.commit(ref) + return false unless commit + commit.sha == sha + end + + def triggered? + trigger_requests.any? + end + + def create_builds(user, trigger_request = nil) + ## + # We persist pipeline only if there are builds available + # + return unless config_processor + + build_builds_for_stages(config_processor.stages, user, + 'success', trigger_request) && save + end + + def create_next_builds(build) + return unless config_processor + + # don't create other builds if this one is retried + latest_builds = builds.latest + return unless latest_builds.exists?(build.id) + + # get list of stages after this build + next_stages = config_processor.stages.drop_while { |stage| stage != build.stage } + next_stages.delete(build.stage) + + # get status for all prior builds + prior_builds = latest_builds.where.not(stage: next_stages) + prior_status = prior_builds.status + + # build builds for next stage that has builds available + # and save pipeline if we have builds + build_builds_for_stages(next_stages, build.user, prior_status, + build.trigger_request) && save + end + + def retried + @retried ||= (statuses.order(id: :desc) - statuses.latest) + end + + def coverage + coverage_array = statuses.latest.map(&:coverage).compact + if coverage_array.size >= 1 + '%.2f' % (coverage_array.reduce(:+) / coverage_array.size) + end + end + + def config_processor + return nil unless ci_yaml_file + return @config_processor if defined?(@config_processor) + + @config_processor ||= begin + Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.path_with_namespace) + rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e + self.yaml_errors = e.message + nil + rescue + self.yaml_errors = 'Undefined error' + nil + end + end + + def ci_yaml_file + return @ci_yaml_file if defined?(@ci_yaml_file) + + @ci_yaml_file ||= begin + blob = project.repository.blob_at(sha, '.gitlab-ci.yml') + blob.load_all_data!(project.repository) + blob.data + rescue + nil + end + end + + def skip_ci? + git_commit_message =~ /(\[ci skip\])/ if git_commit_message + end + + def environments + builds.where.not(environment: nil).success.pluck(:environment).uniq + end + + def notes + Note.for_commit_id(sha) + end + + private + + def build_builds_for_stages(stages, user, status, trigger_request) + ## + # Note that `Array#any?` implements a short circuit evaluation, so we + # build builds only for the first stage that has builds available. + # + stages.any? do |stage| + CreateBuildsService.new(self) + .execute(stage, user, status, trigger_request).present? + end + end + + def update_state + statuses.reload + self.status = if yaml_errors.blank? + statuses.latest.status || 'skipped' + else + 'failed' + end + self.started_at = statuses.started_at + self.finished_at = statuses.finished_at + self.duration = statuses.latest.duration + save + end + end +end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 90349a07594..adb65292208 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -1,28 +1,10 @@ -# == Schema Information -# -# Table name: ci_runners -# -# id :integer not null, primary key -# token :string(255) -# created_at :datetime -# updated_at :datetime -# description :string(255) -# contacted_at :datetime -# active :boolean default(TRUE), not null -# is_shared :boolean default(FALSE) -# name :string(255) -# version :string(255) -# revision :string(255) -# platform :string(255) -# architecture :string(255) -# - module Ci class Runner < ActiveRecord::Base extend Ci::Model LAST_CONTACT_TIME = 5.minutes.ago - AVAILABLE_SCOPES = ['specific', 'shared', 'active', 'paused', 'online'] + AVAILABLE_SCOPES = %w[specific shared active paused online] + FORM_EDITABLE = %i[description tag_list active run_untagged] has_many :builds, class_name: 'Ci::Build' has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject' @@ -44,6 +26,8 @@ module Ci .where("ci_runner_projects.gl_project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id) end + validate :tag_constraints + acts_as_taggable # Searches for runners matching the given query. @@ -76,7 +60,7 @@ module Ci end def display_name - return short_sha unless !description.blank? + return short_sha if description.blank? description end @@ -114,5 +98,18 @@ module Ci def short_sha token[0...8] if token end + + def has_tags? + tag_list.any? + end + + private + + def tag_constraints + unless has_tags? || run_untagged? + errors.add(:tags_list, + 'can not be empty when runner is not allowed to pick untagged jobs') + end + end end end diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index 7b16f207a26..4b44ffa886e 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -1,15 +1,3 @@ -# == Schema Information -# -# Table name: ci_runner_projects -# -# id :integer not null, primary key -# runner_id :integer not null -# project_id :integer -# created_at :datetime -# updated_at :datetime -# gl_project_id :integer -# - module Ci class RunnerProject < ActiveRecord::Base extend Ci::Model diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index 2b9a457c8ab..a0b19b51a12 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -1,16 +1,3 @@ -# == Schema Information -# -# Table name: ci_triggers -# -# id :integer not null, primary key -# token :string(255) -# project_id :integer -# deleted_at :datetime -# created_at :datetime -# updated_at :datetime -# gl_project_id :integer -# - module Ci class Trigger < ActiveRecord::Base extend Ci::Model diff --git a/app/models/ci/trigger_request.rb b/app/models/ci/trigger_request.rb index 9973d2e5ade..b69ae37668c 100644 --- a/app/models/ci/trigger_request.rb +++ b/app/models/ci/trigger_request.rb @@ -1,21 +1,9 @@ -# == Schema Information -# -# Table name: ci_trigger_requests -# -# id :integer not null, primary key -# trigger_id :integer not null -# variables :text -# created_at :datetime -# updated_at :datetime -# commit_id :integer -# - module Ci class TriggerRequest < ActiveRecord::Base extend Ci::Model belongs_to :trigger, class_name: 'Ci::Trigger' - belongs_to :commit, class_name: 'Ci::Commit' + belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id has_many :builds, class_name: 'Ci::Build' serialize :variables diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index e786bd7dd93..f8d5d4486fd 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -1,17 +1,3 @@ -# == Schema Information -# -# Table name: ci_variables -# -# id :integer not null, primary key -# project_id :integer -# key :string(255) -# value :text -# encrypted_value :text -# encrypted_value_salt :string(255) -# encrypted_value_iv :string(255) -# gl_project_id :integer -# - module Ci class Variable < ActiveRecord::Base extend Ci::Model @@ -25,6 +11,9 @@ module Ci format: { with: /\A[a-zA-Z0-9_]+\z/, message: "can contain only letters, digits and '_'." } - attr_encrypted :value, mode: :per_attribute_iv_and_salt, key: Gitlab::Application.secrets.db_key_base + attr_encrypted :value, + mode: :per_attribute_iv_and_salt, + key: Gitlab::Application.secrets.db_key_base, + algorithm: 'aes-256-cbc' end end |