diff options
Diffstat (limited to 'app/models/ci')
-rw-r--r-- | app/models/ci/build.rb | 22 | ||||
-rw-r--r-- | app/models/ci/build_metadata.rb | 1 | ||||
-rw-r--r-- | app/models/ci/build_need.rb | 2 | ||||
-rw-r--r-- | app/models/ci/build_trace.rb | 26 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/redis.rb | 5 | ||||
-rw-r--r-- | app/models/ci/instance_variable.rb | 8 | ||||
-rw-r--r-- | app/models/ci/job_artifact.rb | 53 | ||||
-rw-r--r-- | app/models/ci/pipeline.rb | 144 | ||||
-rw-r--r-- | app/models/ci/pipeline_enums.rb | 5 | ||||
-rw-r--r-- | app/models/ci/pipeline_message.rb | 25 | ||||
-rw-r--r-- | app/models/ci/ref.rb | 2 | ||||
-rw-r--r-- | app/models/ci/runner.rb | 4 | ||||
-rw-r--r-- | app/models/ci/stage.rb | 6 | ||||
-rw-r--r-- | app/models/ci/variable.rb | 2 |
14 files changed, 230 insertions, 75 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b5e68b55f72..6c90645e997 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -27,7 +27,7 @@ module Ci upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? }, refspecs: -> (build) { build.merge_request_ref? }, artifacts_exclude: -> (build) { build.supports_artifacts_exclude? }, - release_steps: -> (build) { build.release_steps? } + multi_build_steps: -> (build) { build.multi_build_steps? } }.freeze DEFAULT_RETRIES = { @@ -539,7 +539,6 @@ module Ci .concat(job_variables) .concat(environment_changed_page_variables) .concat(persisted_environment_variables) - .concat(deploy_freeze_variables) .to_runner_variables end end @@ -595,18 +594,6 @@ module Ci end end - def deploy_freeze_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables unless freeze_period? - - variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') - end - end - - def freeze_period? - Ci::FreezePeriodStatus.new(project: project).execute - end - def dependency_variables return [] if all_dependencies.empty? @@ -801,6 +788,11 @@ module Ci has_expiring_artifacts? && job_artifacts_archive.present? end + def self.keep_artifacts! + update_all(artifacts_expire_at: nil) + Ci::JobArtifact.where(job: self.select(:id)).update_all(expire_at: nil) + end + def keep_artifacts! self.update(artifacts_expire_at: nil) self.job_artifacts.update_all(expire_at: nil) @@ -885,7 +877,7 @@ module Ci Gitlab::Ci::Features.artifacts_exclude_enabled? end - def release_steps? + def multi_build_steps? options.dig(:release)&.any? && Gitlab::Ci::Features.release_generation_enabled? end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 0df5ebfe843..4094bdb26dc 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -19,6 +19,7 @@ module Ci before_create :set_build_project validates :build, presence: true + validates :secrets, json_schema: { filename: 'build_metadata_secrets' } serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize serialize :config_variables, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index 0b243c20e67..b977a5f4419 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -4,6 +4,8 @@ module Ci class BuildNeed < ApplicationRecord extend Gitlab::Ci::Model + include BulkInsertSafe + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :needs validates :build, presence: true diff --git a/app/models/ci/build_trace.rb b/app/models/ci/build_trace.rb index b9db1559836..f70e1ed69ea 100644 --- a/app/models/ci/build_trace.rb +++ b/app/models/ci/build_trace.rb @@ -2,40 +2,22 @@ module Ci class BuildTrace - CONVERTERS = { - html: Gitlab::Ci::Ansi2html, - json: Gitlab::Ci::Ansi2json - }.freeze - attr_reader :trace, :build delegate :state, :append, :truncated, :offset, :size, :total, to: :trace, allow_nil: true delegate :id, :status, :complete?, to: :build, prefix: true - def initialize(build:, stream:, state:, content_format:) + def initialize(build:, stream:, state:) @build = build - @content_format = content_format if stream.valid? stream.limit - @trace = CONVERTERS.fetch(content_format).convert(stream.stream, state) + @trace = Gitlab::Ci::Ansi2json.convert(stream.stream, state) end end - def json? - @content_format == :json - end - - def html? - @content_format == :html - end - - def json_lines - @trace&.lines if json? - end - - def html_lines - @trace&.html if html? + def lines + @trace&.lines end end end diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb index 813eaf5d839..c3864f78b01 100644 --- a/app/models/ci/build_trace_chunks/redis.rb +++ b/app/models/ci/build_trace_chunks/redis.rb @@ -35,7 +35,10 @@ module Ci keys = keys.map { |key| key_raw(*key) } Gitlab::Redis::SharedState.with do |redis| - redis.del(keys) + # https://gitlab.com/gitlab-org/gitlab/-/issues/224171 + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.del(keys) + end end end diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb index 8245729a884..628749b32cb 100644 --- a/app/models/ci/instance_variable.rb +++ b/app/models/ci/instance_variable.rb @@ -45,13 +45,5 @@ module Ci end end end - - private - - def validate_plan_limit_not_exceeded - if Gitlab::Ci::Features.instance_level_variables_limit_enabled? - super - end - end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 8aba9356949..dbeba1ece31 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -7,10 +7,13 @@ module Ci include UpdateProjectStatistics include UsageStatistics include Sortable + include IgnorableColumns extend Gitlab::Ci::Model NotSupportedAdapterError = Class.new(StandardError) + ignore_columns :locked, remove_after: '2020-07-22', remove_with: '13.4' + TEST_REPORT_FILE_TYPES = %w[junit].freeze COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze @@ -34,13 +37,16 @@ module Ci license_management: 'gl-license-management-report.json', license_scanning: 'gl-license-scanning-report.json', performance: 'performance.json', + browser_performance: 'browser-performance.json', + load_performance: 'load-performance.json', metrics: 'metrics.txt', lsif: 'lsif.json', dotenv: '.env', cobertura: 'cobertura-coverage.xml', terraform: 'tfplan.json', cluster_applications: 'gl-cluster-applications.json', - requirements: 'requirements.json' + requirements: 'requirements.json', + coverage_fuzzing: 'gl-coverage-fuzzing.json' }.freeze INTERNAL_TYPES = { @@ -72,8 +78,11 @@ module Ci license_management: :raw, license_scanning: :raw, performance: :raw, + browser_performance: :raw, + load_performance: :raw, terraform: :raw, - requirements: :raw + requirements: :raw, + coverage_fuzzing: :raw }.freeze DOWNLOADABLE_TYPES = %w[ @@ -91,6 +100,8 @@ module Ci lsif metrics performance + browser_performance + load_performance sast secret_detection requirements @@ -98,9 +109,7 @@ module Ci TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze - # This is required since we cannot add a default to the database - # https://gitlab.com/gitlab-org/gitlab/-/issues/215418 - attribute :locked, :boolean, default: false + PLAN_LIMIT_PREFIX = 'ci_max_artifact_size_' belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id @@ -117,10 +126,9 @@ module Ci after_save :update_file_store, if: :saved_change_to_file? scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } - scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } + scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } - scope :for_ref, ->(ref, project_id) { joins(job: :pipeline).where(ci_pipelines: { ref: ref, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } scope :with_file_types, -> (file_types) do @@ -157,8 +165,7 @@ module Ci scope :expired, -> (limit) { where('expire_at < ?', Time.current).limit(limit) } scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } - scope :locked, -> { where(locked: true) } - scope :unlocked, -> { where(locked: [false, nil]) } + scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked).order(expire_at: :desc) } scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } @@ -176,7 +183,7 @@ module Ci codequality: 9, ## EE-specific license_management: 10, ## EE-specific license_scanning: 101, ## EE-specific till 13.0 - performance: 11, ## EE-specific + performance: 11, ## EE-specific till 13.2 metrics: 12, ## EE-specific metrics_referee: 13, ## runner referees network_referee: 14, ## runner referees @@ -187,7 +194,10 @@ module Ci accessibility: 19, cluster_applications: 20, secret_detection: 21, ## EE-specific - requirements: 22 ## EE-specific + requirements: 22, ## EE-specific + coverage_fuzzing: 23, ## EE-specific + browser_performance: 24, ## EE-specific + load_performance: 25 ## EE-specific } enum file_format: { @@ -235,6 +245,12 @@ module Ci self.update_column(:file_store, file.object_store) end + def self.associated_file_types_for(file_type) + return unless file_types.include?(file_type) + + [file_type] + end + def self.total_size self.sum(:size) end @@ -286,6 +302,21 @@ module Ci where(job_id: job_id).trace.take&.file&.file&.exists? end + def self.max_artifact_size(type:, project:) + max_size = if Feature.enabled?(:ci_max_artifact_size_per_type, project, default_enabled: false) + limit_name = "#{PLAN_LIMIT_PREFIX}#{type}" + + project.actual_limits.limit_for( + limit_name, + alternate_limit: -> { project.closest_setting(:max_artifacts_size) } + ) + else + project.closest_setting(:max_artifacts_size) + end + + max_size&.megabytes.to_i + end + private def file_format_adapter_class diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 497e1a4d74a..d4b439d648f 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -3,7 +3,7 @@ module Ci class Pipeline < ApplicationRecord extend Gitlab::Ci::Model - include HasStatus + include Ci::HasStatus include Importable include AfterCommitQueue include Presentable @@ -51,6 +51,8 @@ module Ci has_many :latest_builds, -> { latest }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' has_many :downloadable_artifacts, -> { not_expired.downloadable }, through: :latest_builds, source: :job_artifacts + has_many :messages, class_name: 'Ci::PipelineMessage', inverse_of: :pipeline + # Merge requests for which the current pipeline is running against # the merge request's latest commit. has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest' @@ -80,6 +82,7 @@ module Ci has_one :pipeline_config, class_name: 'Ci::PipelineConfig', inverse_of: :pipeline has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id + has_many :latest_builds_report_results, through: :latest_builds, source: :report_results accepts_nested_attributes_for :variables, reject_if: :persisted? @@ -110,6 +113,8 @@ module Ci # extend this `Hash` with new values. enum failure_reason: ::Ci::PipelineEnums.failure_reasons + enum locked: { unlocked: 0, artifacts_locked: 1 } + state_machine :status, initial: :created do event :enqueue do transition [:created, :manual, :waiting_for_resource, :preparing, :skipped, :scheduled] => :pending @@ -244,6 +249,14 @@ module Ci pipeline.run_after_commit { AutoDevops::DisableWorker.perform_async(pipeline.id) } end + + after_transition any => [:success] do |pipeline| + next unless Gitlab::Ci::Features.keep_latest_artifacts_for_ref_enabled?(pipeline.project) + + pipeline.run_after_commit do + Ci::PipelineSuccessUnlockArtifactsWorker.perform_async(pipeline.id) + end + end end scope :internal, -> { where(source: internal_sources) } @@ -256,7 +269,14 @@ module Ci scope :for_ref, -> (ref) { where(ref: ref) } scope :for_id, -> (id) { where(id: id) } scope :for_iid, -> (iid) { where(iid: iid) } + scope :for_project, -> (project) { where(project: project) } scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } + scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } + scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } + + scope :outside_pipeline_family, ->(pipeline) do + where.not(id: pipeline.same_family_pipeline_ids) + end scope :with_reports, -> (reports_scope) do where('EXISTS (?)', ::Ci::Build.latest.with_reports(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1)) @@ -270,6 +290,15 @@ module Ci ) end + # Returns the pipelines that associated with the given merge request. + # In general, please use `Ci::PipelinesForMergeRequestFinder` instead, + # for checking permission of the actor. + scope :triggered_by_merge_request, -> (merge_request) do + ci_sources.where(source: :merge_request_event, + merge_request: merge_request, + project: [merge_request.source_project, merge_request.target_project]) + end + # Returns the pipelines in descending order (= newest first), optionally # limited to a number of references. # @@ -348,6 +377,10 @@ module Ci success.group(:project_id).select('max(id) as id') end + def self.last_finished_for_ref_id(ci_ref_id) + where(ci_ref_id: ci_ref_id).ci_sources.finished.order(id: :desc).select(:id).take + end + def self.truncate_sha(sha) sha[0...8] end @@ -440,6 +473,10 @@ module Ci end end + def triggered_pipelines_with_preloads + triggered_pipelines.preload(:source_job) + end + def legacy_stages if ::Gitlab::Ci::Features.composite_status?(project) legacy_stages_using_composite_status @@ -552,10 +589,28 @@ module Ci end end + def lazy_ref_commit + return unless ::Gitlab::Ci::Features.pipeline_latest? + + BatchLoader.for(ref).batch do |refs, loader| + next unless project.repository_exists? + + project.repository.list_commits_by_ref_name(refs).then do |commits| + commits.each { |key, commit| loader.call(key, commits[key]) } + end + end + end + def latest? return false unless git_ref && commit.present? - project.commit(git_ref) == commit + unless ::Gitlab::Ci::Features.pipeline_latest? + return project.commit(git_ref) == commit + end + + return false if lazy_ref_commit.nil? + + lazy_ref_commit.id == commit.id end def retried @@ -569,10 +624,46 @@ module Ci end end + def batch_lookup_report_artifact_for_file_type(file_type) + latest_report_artifacts + .values_at(*::Ci::JobArtifact.associated_file_types_for(file_type.to_s)) + .flatten + .compact + .last + end + + # This batch loads the latest reports for each CI job artifact + # type (e.g. sast, dast, etc.) in a single SQL query to eliminate + # the need to do N different `job_artifacts.where(file_type: + # X).last` calls. + # + # Return a hash of file type => array of 1 job artifact + def latest_report_artifacts + ::Gitlab::SafeRequestStore.fetch("pipeline:#{self.id}:latest_report_artifacts") do + # Note we use read_attribute(:project_id) to read the project + # ID instead of self.project_id. The latter appears to load + # the Project model. This extra filter doesn't appear to + # affect query plan but included to ensure we don't leak the + # wrong informaiton. + ::Ci::JobArtifact.where( + id: job_artifacts.with_reports + .select('max(ci_job_artifacts.id) as id') + .where(project_id: self.read_attribute(:project_id)) + .group(:file_type) + ) + .preload(:job) + .group_by(&:file_type) + end + end + def has_kubernetes_active? project.deployment_platform&.active? end + def freeze_period? + Ci::FreezePeriodStatus.new(project: project).execute + end + def has_warnings? number_of_warnings.positive? end @@ -607,6 +698,25 @@ module Ci yaml_errors.present? end + def add_error_message(content) + add_message(:error, content) + end + + def add_warning_message(content) + add_message(:warning, content) + end + + # We can't use `messages.error` scope here because messages should also be + # read when the pipeline is not persisted. Using the scope will return no + # results as it would query persisted data. + def error_messages + messages.select(&:error?) + end + + def warning_messages + messages.select(&:warning?) + end + # Manually set the notes for a Ci::Pipeline # There is no ActiveRecord relation between Ci::Pipeline and notes # as they are related to a commit sha. This method helps importing @@ -639,7 +749,7 @@ module Ci when 'manual' then block when 'scheduled' then delay else - raise HasStatus::UnknownStatusError, + raise Ci::HasStatus::UnknownStatusError, "Unknown status `#{new_status}`" end end @@ -683,6 +793,7 @@ module Ci end variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if has_kubernetes_active? + variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if freeze_period? if external_pull_request_event? && external_pull_request variables.concat(external_pull_request.predefined_variables) @@ -748,13 +859,10 @@ module Ci end # If pipeline is a child of another pipeline, include the parent - # and the siblings, otherwise return only itself. + # and the siblings, otherwise return only itself and children. def same_family_pipeline_ids - if (parent = parent_pipeline) - [parent.id] + parent.child_pipelines.pluck(:id) - else - [self.id] - end + parent = parent_pipeline || self + [parent.id] + parent.child_pipelines.pluck(:id) end def bridge_triggered? @@ -802,6 +910,10 @@ module Ci complete? && latest_report_builds(reports_scope).exists? end + def test_report_summary + Gitlab::Ci::Reports::TestReportSummary.new(latest_builds_report_results) + end + def test_reports Gitlab::Ci::Reports::TestReports.new.tap do |test_reports| latest_report_builds(Ci::JobArtifact.test_reports).preload(:project).find_each do |build| @@ -840,6 +952,10 @@ module Ci end end + def has_archive_artifacts? + complete? && builds.latest.with_existing_job_artifacts(Ci::JobArtifact.archive.or(Ci::JobArtifact.metadata)).exists? + end + def has_exposed_artifacts? complete? && builds.latest.with_exposed_artifacts.exists? end @@ -925,7 +1041,7 @@ module Ci stages.find_by!(name: name) end - def error_messages + def full_error_messages errors ? errors.full_messages.to_sentence : "" end @@ -964,8 +1080,6 @@ module Ci # Set scheduling type of processables if they were created before scheduling_type # data was deployed (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22246). def ensure_scheduling_type! - return unless ::Gitlab::Ci::Features.ensure_scheduling_type_enabled? - processables.populate_scheduling_type! end @@ -977,6 +1091,12 @@ module Ci private + def add_message(severity, content) + return unless Gitlab::Ci::Features.store_pipeline_messages?(project) + + messages.build(severity: severity, content: content) + end + def pipeline_data Gitlab::DataBuilder::Pipeline.build(self) end diff --git a/app/models/ci/pipeline_enums.rb b/app/models/ci/pipeline_enums.rb index 2ccd8445aa8..352dc56aac7 100644 --- a/app/models/ci/pipeline_enums.rb +++ b/app/models/ci/pipeline_enums.rb @@ -31,7 +31,7 @@ module Ci merge_request_event: 10, external_pull_request_event: 11, parent_pipeline: 12, - ondemand_scan: 13 + ondemand_dast_scan: 13 } end @@ -45,7 +45,8 @@ module Ci webide_source: 3, remote_source: 4, external_project_source: 5, - bridge_source: 6 + bridge_source: 6, + parameter_source: 7 } end diff --git a/app/models/ci/pipeline_message.rb b/app/models/ci/pipeline_message.rb new file mode 100644 index 00000000000..a47ec554462 --- /dev/null +++ b/app/models/ci/pipeline_message.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ci + class PipelineMessage < ApplicationRecord + extend Gitlab::Ci::Model + + MAX_CONTENT_LENGTH = 10_000 + + belongs_to :pipeline + + validates :content, presence: true + + before_save :truncate_long_content + + enum severity: { error: 0, warning: 1 } + + private + + def truncate_long_content + return if content.length <= MAX_CONTENT_LENGTH + + self.content = content.truncate(MAX_CONTENT_LENGTH) + end + end +end diff --git a/app/models/ci/ref.rb b/app/models/ci/ref.rb index be6062b6e6e..29b44575d65 100644 --- a/app/models/ci/ref.rb +++ b/app/models/ci/ref.rb @@ -43,7 +43,7 @@ module Ci end def last_finished_pipeline_id - Ci::Pipeline.where(ci_ref_id: self.id).finished.order(id: :desc).select(:id).take&.id + Ci::Pipeline.last_finished_for_ref_id(self.id)&.id end def update_status_by!(pipeline) diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8fc273556f0..1cd6c64841b 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -239,6 +239,10 @@ module Ci runner_projects.count == 1 end + def belongs_to_more_than_one_project? + self.projects.limit(2).count(:all) > 1 + end + def assigned_to_group? runner_namespaces.any? end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index a316b4718e0..41215601704 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -4,10 +4,10 @@ module Ci class Stage < ApplicationRecord extend Gitlab::Ci::Model include Importable - include HasStatus + include Ci::HasStatus include Gitlab::OptimisticLocking - enum status: HasStatus::STATUSES_ENUM + enum status: Ci::HasStatus::STATUSES_ENUM belongs_to :project belongs_to :pipeline @@ -98,7 +98,7 @@ module Ci when 'scheduled' then delay when 'skipped', nil then skip else - raise HasStatus::UnknownStatusError, + raise Ci::HasStatus::UnknownStatusError, "Unknown status `#{new_status}`" end end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 08d39595c61..13358b95a47 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -18,5 +18,7 @@ module Ci } scope :unprotected, -> { where(protected: false) } + scope :by_key, -> (key) { where(key: key) } + scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) } end end |