diff options
Diffstat (limited to 'app/models/ci')
| -rw-r--r-- | app/models/ci/artifact_blob.rb | 2 | ||||
| -rw-r--r-- | app/models/ci/build.rb | 19 | ||||
| -rw-r--r-- | app/models/ci/build_trace.rb | 41 | ||||
| -rw-r--r-- | app/models/ci/build_trace_section.rb | 3 | ||||
| -rw-r--r-- | app/models/ci/group.rb | 19 | ||||
| -rw-r--r-- | app/models/ci/job_artifact.rb | 6 | ||||
| -rw-r--r-- | app/models/ci/legacy_stage.rb | 14 | ||||
| -rw-r--r-- | app/models/ci/persistent_ref.rb | 44 | ||||
| -rw-r--r-- | app/models/ci/pipeline.rb | 93 | ||||
| -rw-r--r-- | app/models/ci/pipeline_enums.rb | 1 | ||||
| -rw-r--r-- | app/models/ci/pipeline_schedule.rb | 2 | ||||
| -rw-r--r-- | app/models/ci/sources/pipeline.rb | 25 | ||||
| -rw-r--r-- | app/models/ci/stage.rb | 9 | ||||
| -rw-r--r-- | app/models/ci/trigger.rb | 2 |
14 files changed, 235 insertions, 45 deletions
diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb index ef00ad75683..76d4b9d6206 100644 --- a/app/models/ci/artifact_blob.rb +++ b/app/models/ci/artifact_blob.rb @@ -53,7 +53,7 @@ module Ci pages_config.enabled && pages_config.artifacts_server && EXTENSIONS_SERVED_BY_PAGES.include?(File.extname(name)) && - job.project.public? + (pages_config.access_control || job.project.public?) end private diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1f8a0373450..c48ab28ce73 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -12,7 +12,6 @@ module Ci include Presentable include Importable include Gitlab::Utils::StrongMemoize - include Deployable include HasRef BuildArchivedError = Class.new(StandardError) @@ -43,6 +42,7 @@ module Ci has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id Ci::JobArtifact.file_types.each do |key, value| has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id @@ -118,8 +118,6 @@ module Ci scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } - scope :with_artifacts_stored_locally, -> { with_existing_job_artifacts(Ci::JobArtifact.archive.with_files_stored_locally) } - scope :with_archived_trace_stored_locally, -> { with_existing_job_artifacts(Ci::JobArtifact.trace.with_files_stored_locally) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } @@ -130,6 +128,12 @@ module Ci scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) } scope :finished_before, -> (date) { finished.where('finished_at < ?', date) } + scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") } + + scope :with_secure_reports_from_config_options, -> (job_types) do + joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) + end + scope :matches_tag_ids, -> (tag_ids) do matcher = ::ActsAsTaggableOn::Tagging .where(taggable_type: CommitStatus.name) @@ -236,6 +240,7 @@ module Ci end after_transition pending: :running do |build| + build.pipeline.persistent_ref.create build.deployment&.run build.run_after_commit do @@ -412,10 +417,6 @@ module Ci self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options end - def has_deployment? - !!self.deployment - end - def outdated_deployment? success? && !deployment.try(:last?) end @@ -753,6 +754,10 @@ module Ci true end + def invalid_dependencies + dependencies.reject(&:valid_dependency?) + end + def runner_required_feature_names strong_memoize(:runner_required_feature_names) do RUNNER_FEATURES.select do |feature, method| diff --git a/app/models/ci/build_trace.rb b/app/models/ci/build_trace.rb new file mode 100644 index 00000000000..b9db1559836 --- /dev/null +++ b/app/models/ci/build_trace.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Ci + class BuildTrace + CONVERTERS = { + html: Gitlab::Ci::Ansi2html, + json: Gitlab::Ci::Ansi2json + }.freeze + + attr_reader :trace, :build + + delegate :state, :append, :truncated, :offset, :size, :total, to: :trace, allow_nil: true + delegate :id, :status, :complete?, to: :build, prefix: true + + def initialize(build:, stream:, state:, content_format:) + @build = build + @content_format = content_format + + if stream.valid? + stream.limit + @trace = CONVERTERS.fetch(content_format).convert(stream.stream, state) + end + end + + def json? + @content_format == :json + end + + def html? + @content_format == :html + end + + def json_lines + @trace&.lines if json? + end + + def html_lines + @trace&.html if html? + end + end +end diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb index 8be42eb48d6..7fe6b753da1 100644 --- a/app/models/ci/build_trace_section.rb +++ b/app/models/ci/build_trace_section.rb @@ -4,6 +4,9 @@ module Ci class BuildTraceSection < ApplicationRecord extend Gitlab::Ci::Model + # Only remove > 2019-11-22 and > 12.5 + self.ignored_columns += %i[id] + belongs_to :build, class_name: 'Ci::Build' belongs_to :project belongs_to :section_name, class_name: 'Ci::BuildTraceSectionName' diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 9b2c3c807ac..0e05318b253 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -9,6 +9,7 @@ module Ci # class Group include StaticModel + include Gitlab::Utils::StrongMemoize attr_reader :stage, :name, :jobs @@ -21,7 +22,17 @@ module Ci end def status - @status ||= commit_statuses.status + strong_memoize(:status) do + if Feature.enabled?(:ci_composite_status, default_enabled: false) + Gitlab::Ci::Status::Composite + .new(@jobs) + .status + else + CommitStatus + .where(id: @jobs) + .legacy_status + end + end end def detailed_status(current_user) @@ -40,11 +51,5 @@ module Ci self.new(stage, name: group_name, jobs: grouped_statuses) end end - - private - - def commit_statuses - @commit_statuses ||= CommitStatus.where(id: jobs.map(&:id)) - end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index da2758507ce..62bf2c3ac9c 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -5,6 +5,7 @@ module Ci include AfterCommitQueue include ObjectStorage::BackgroundMove include UpdateProjectStatistics + include Sortable extend Gitlab::Ci::Model NotSupportedAdapterError = Class.new(StandardError) @@ -64,6 +65,7 @@ module Ci after_save :update_file_store, if: :saved_change_to_file? scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } + scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :with_file_types, -> (file_types) do types = self.file_types.select { |file_type| file_types.include?(file_type) }.values @@ -143,6 +145,10 @@ module Ci self.update_column(:file_store, file.object_store) end + def self.total_size + self.sum(:size) + end + def self.artifacts_size_for(project) self.where(project: project).sum(:size) end diff --git a/app/models/ci/legacy_stage.rb b/app/models/ci/legacy_stage.rb index 930c8a71453..2fd369c9aff 100644 --- a/app/models/ci/legacy_stage.rb +++ b/app/models/ci/legacy_stage.rb @@ -14,7 +14,8 @@ module Ci @pipeline = pipeline @name = name @status = status - @warnings = warnings + # support ints and booleans + @has_warnings = ActiveRecord::Type::Boolean.new.cast(warnings) end def groups @@ -30,7 +31,7 @@ module Ci end def status - @status ||= statuses.latest.status + @status ||= statuses.latest.slow_composite_status end def detailed_status(current_user) @@ -52,11 +53,12 @@ module Ci end def has_warnings? - if @warnings.is_a?(Integer) - @warnings > 0 - else - statuses.latest.failed_but_allowed.any? + # lazilly calculate the warnings + if @has_warnings.nil? + @has_warnings = statuses.latest.failed_but_allowed.any? end + + @has_warnings end def manual_playable? diff --git a/app/models/ci/persistent_ref.rb b/app/models/ci/persistent_ref.rb new file mode 100644 index 00000000000..be3d4aa3203 --- /dev/null +++ b/app/models/ci/persistent_ref.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Ci + ## + # The persistent pipeline ref to ensure runners can safely fetch source code + # even if force-push/source-branch-deletion happens. + class PersistentRef + include ActiveModel::Model + + attr_accessor :pipeline + + delegate :project, :sha, to: :pipeline + delegate :repository, to: :project + delegate :ref_exists?, :create_ref, :delete_refs, to: :repository + + def exist? + ref_exists?(path) + rescue + false + end + + def create + return if exist? + + create_ref(sha, path) + rescue => e + Gitlab::Sentry + .track_acceptable_exception(e, extra: { pipeline_id: pipeline.id }) + end + + def delete + delete_refs(path) + rescue Gitlab::Git::Repository::NoRepository + # no-op + rescue => e + Gitlab::Sentry + .track_acceptable_exception(e, extra: { pipeline_id: pipeline.id }) + end + + def path + "refs/#{Repository::REF_PIPELINES}/#{pipeline.id}" + end + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 20b8be4017e..3bf19399cec 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -25,7 +25,7 @@ module Ci belongs_to :merge_request, class_name: 'MergeRequest' belongs_to :external_pull_request - has_internal_id :iid, scope: :project, presence: false, init: ->(s) do + has_internal_id :iid, scope: :project, presence: false, ensure_if: -> { !importing? }, init: ->(s) do s&.project&.all_pipelines&.maximum(:iid) || s&.project&.all_pipelines&.count end @@ -52,9 +52,15 @@ module Ci has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_pipeline_id + has_one :source_pipeline, class_name: 'Ci::Sources::Pipeline', inverse_of: :pipeline has_one :chat_data, class_name: 'Ci::PipelineChatData' + has_many :triggered_pipelines, through: :sourced_pipelines, source: :pipeline + has_one :triggered_by_pipeline, through: :source_pipeline, source: :source_pipeline + has_one :source_job, through: :source_pipeline, source: :source_job + accepts_nested_attributes_for :variables, reject_if: :persisted? delegate :id, to: :project, prefix: true @@ -174,6 +180,8 @@ module Ci after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline| pipeline.run_after_commit do + pipeline.persistent_ref.delete + pipeline.all_merge_requests.each do |merge_request| next unless merge_request.auto_merge_enabled? @@ -209,6 +217,8 @@ module Ci scope :for_sha, -> (sha) { where(sha: sha) } scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) } scope :for_sha_or_source_sha, -> (sha) { for_sha(sha).or(for_source_sha(sha)) } + scope :for_ref, -> (ref) { where(ref: ref) } + scope :for_id, -> (id) { where(id: id) } scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :triggered_by_merge_request, -> (merge_request) do @@ -279,16 +289,16 @@ module Ci end end - # Returns a Hash containing the latest pipeline status for every given + # Returns a Hash containing the latest pipeline for every given # commit. # - # The keys of this Hash are the commit SHAs, the values the statuses. + # The keys of this Hash are the commit SHAs, the values the pipelines. # - # commits - The list of commit SHAs to get the status for. + # commits - The list of commit SHAs to get the pipelines for. # ref - The ref to scope the data to (e.g. "master"). If the ref is not - # given we simply get the latest status for the commits, regardless - # of what refs their pipelines belong to. - def self.latest_status_per_commit(commits, ref = nil) + # given we simply get the latest pipelines for the commits, regardless + # of what refs the pipelines belong to. + def self.latest_pipeline_per_commit(commits, ref = nil) p1 = arel_table p2 = arel_table.alias @@ -302,15 +312,14 @@ module Ci cond = cond.and(p1[:ref].eq(p2[:ref])) if ref join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond) - relation = select(:sha, :status) - .where(sha: commits) + relation = where(sha: commits) .where(p2[:id].eq(nil)) .joins(join.join_sources) relation = relation.where(ref: ref) if ref - relation.each_with_object({}) do |row, hash| - hash[row[:sha]] = row[:status] + relation.each_with_object({}) do |pipeline, hash| + hash[pipeline.sha] = pipeline end end @@ -385,13 +394,12 @@ module Ci end end - def legacy_stages + def legacy_stages_using_sql # TODO, this needs refactoring, see gitlab-foss#26481. - stages_query = statuses .group('stage').select(:stage).order('max(stage_idx)') - status_sql = statuses.latest.where('stage=sg.stage').status_sql + status_sql = statuses.latest.where('stage=sg.stage').legacy_status_sql warnings_sql = statuses.latest.select('COUNT(*)') .where('stage=sg.stage').failed_but_allowed.to_sql @@ -404,6 +412,30 @@ module Ci end end + def legacy_stages_using_composite_status + stages = statuses.latest + .order(:stage_idx, :stage) + .group_by(&:stage) + + stages.map do |stage_name, jobs| + composite_status = Gitlab::Ci::Status::Composite + .new(jobs) + + Ci::LegacyStage.new(self, + name: stage_name, + status: composite_status.status, + warnings: composite_status.warnings?) + end + end + + def legacy_stages + if Feature.enabled?(:ci_composite_status, default_enabled: false) + legacy_stages_using_composite_status + else + legacy_stages_using_sql + end + end + def valid_commit_sha if self.sha == Gitlab::Git::BLANK_SHA self.errors.add(:sha, " cant be 00000000 (branch removal)") @@ -584,11 +616,7 @@ module Ci def ci_yaml_file_path return unless repository_source? || unknown_source? - if project.ci_config_path.blank? - '.gitlab-ci.yml' - else - project.ci_config_path - end + project.ci_config_path.presence || '.gitlab-ci.yml' end def ci_yaml_file @@ -638,7 +666,8 @@ module Ci def update_status retry_optimistic_lock(self) do - case latest_builds_status.to_s + new_status = latest_builds_status.to_s + case new_status when 'created' then nil when 'preparing' then prepare when 'pending' then enqueue @@ -651,7 +680,7 @@ module Ci when 'scheduled' then delay else raise HasStatus::UnknownStatusError, - "Unknown status `#{latest_builds_status}`" + "Unknown status `#{new_status}`" end end end @@ -725,6 +754,10 @@ module Ci end end + def all_merge_requests_by_recency + all_merge_requests.order(id: :desc) + end + def detailed_status(current_user) Gitlab::Ci::Status::Pipeline::Factory .new(self, current_user) @@ -771,6 +804,18 @@ module Ci end end + def all_worktree_paths + strong_memoize(:all_worktree_paths) do + project.repository.ls_files(sha) + end + end + + def top_level_worktree_paths + strong_memoize(:top_level_worktree_paths) do + project.repository.tree(sha).blobs.map(&:path) + end + end + def default_branch? ref == project.default_branch end @@ -845,6 +890,10 @@ module Ci end end + def persistent_ref + @persistent_ref ||= PersistentRef.new(pipeline: self) + end + private def ci_yaml_from_repo @@ -894,7 +943,7 @@ module Ci def latest_builds_status return 'failed' unless yaml_errors.blank? - statuses.latest.status || 'skipped' + statuses.latest.slow_composite_status || 'skipped' end def keep_around_commits diff --git a/app/models/ci/pipeline_enums.rb b/app/models/ci/pipeline_enums.rb index cb92aef4bda..859abc4a0d5 100644 --- a/app/models/ci/pipeline_enums.rb +++ b/app/models/ci/pipeline_enums.rb @@ -22,6 +22,7 @@ module Ci schedule: 4, api: 5, external: 6, + pipeline: 7, chat: 8, merge_request_event: 10, external_pull_request_event: 11 diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 42d4e86fe8d..946241b7d4c 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -86,3 +86,5 @@ module Ci end end end + +Ci::PipelineSchedule.prepend_if_ee('EE::Ci::PipelineSchedule') diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb new file mode 100644 index 00000000000..feaec27281c --- /dev/null +++ b/app/models/ci/sources/pipeline.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ci + module Sources + class Pipeline < ApplicationRecord + self.table_name = "ci_sources_pipelines" + + belongs_to :project, class_name: "Project" + belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :source_pipeline + + belongs_to :source_project, class_name: "Project", foreign_key: :source_project_id + belongs_to :source_job, class_name: "CommitStatus", foreign_key: :source_job_id + belongs_to :source_pipeline, class_name: "Ci::Pipeline", foreign_key: :source_pipeline_id + + validates :project, presence: true + validates :pipeline, presence: true + + validates :source_project, presence: true + validates :source_job, presence: true + validates :source_pipeline, presence: true + end + end +end + +::Ci::Sources::Pipeline.prepend_if_ee('::EE::Ci::Sources::Pipeline') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index d90339d90dc..77ac8bfe875 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -78,7 +78,8 @@ module Ci def update_status retry_optimistic_lock(self) do - case statuses.latest.status + new_status = latest_stage_status.to_s + case new_status when 'created' then nil when 'preparing' then prepare when 'pending' then enqueue @@ -91,7 +92,7 @@ module Ci when 'skipped', nil then skip else raise HasStatus::UnknownStatusError, - "Unknown status `#{statuses.latest.status}`" + "Unknown status `#{new_status}`" end end end @@ -124,5 +125,9 @@ module Ci def manual_playable? blocked? || skipped? end + + def latest_stage_status + statuses.latest.slow_composite_status || 'skipped' + end end end diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index 8792c5cf98b..68548bd2fdc 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -45,3 +45,5 @@ module Ci end end end + +Ci::Trigger.prepend_if_ee('EE::Ci::Trigger') |