diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
tree | c42be41678c2586d49a75cabce89322082698334 /app/models/ci | |
parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) | |
download | gitlab-ce-e8d2c2579383897a1dd7f9debd359abe8ae8373d.tar.gz |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to 'app/models/ci')
-rw-r--r-- | app/models/ci/base_model.rb | 17 | ||||
-rw-r--r-- | app/models/ci/build.rb | 42 | ||||
-rw-r--r-- | app/models/ci/build_dependencies.rb | 14 | ||||
-rw-r--r-- | app/models/ci/build_metadata.rb | 4 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunk.rb | 10 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/fog.rb | 35 | ||||
-rw-r--r-- | app/models/ci/group.rb | 5 | ||||
-rw-r--r-- | app/models/ci/instance_variable.rb | 2 | ||||
-rw-r--r-- | app/models/ci/job_artifact.rb | 8 | ||||
-rw-r--r-- | app/models/ci/job_token/project_scope_link.rb | 4 | ||||
-rw-r--r-- | app/models/ci/pending_build.rb | 42 | ||||
-rw-r--r-- | app/models/ci/pipeline.rb | 71 | ||||
-rw-r--r-- | app/models/ci/runner.rb | 19 |
13 files changed, 184 insertions, 89 deletions
diff --git a/app/models/ci/base_model.rb b/app/models/ci/base_model.rb new file mode 100644 index 00000000000..8fb752ead1d --- /dev/null +++ b/app/models/ci/base_model.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Ci + # TODO: https://gitlab.com/groups/gitlab-org/-/epics/6168 + # + # Do not use this yet outside of `ci_instance_variables`. + # This class is part of a migration to move all CI classes to a new separate database. + # Initially we are only going to be moving the `Ci::InstanceVariable` model and it will be duplicated in the main and CI tables + # Do not extend this class in any other models. + class BaseModel < ::ApplicationRecord + self.abstract_class = true + + if Gitlab::Database.has_config?(:ci) + connects_to database: { writing: :ci, reading: :ci } + end + end +end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index fdfffd9b0cd..4328f3f7a4b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -11,7 +11,6 @@ module Ci include Importable include Ci::HasRef include IgnorableColumns - include TaggableQueries BuildArchivedError = Class.new(StandardError) @@ -136,6 +135,7 @@ module Ci scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } scope :eager_load_job_artifacts_archive, -> { includes(:job_artifacts_archive) } + scope :eager_load_tags, -> { includes(:tags) } scope :eager_load_everything, -> do includes( @@ -178,25 +178,6 @@ module Ci joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) end - scope :matches_tag_ids, -> (tag_ids) do - matcher = ::ActsAsTaggableOn::Tagging - .where(taggable_type: CommitStatus.name) - .where(context: 'tags') - .where('taggable_id = ci_builds.id') - .where.not(tag_id: tag_ids).select('1') - - where("NOT EXISTS (?)", matcher) - end - - scope :with_any_tags, -> do - matcher = ::ActsAsTaggableOn::Tagging - .where(taggable_type: CommitStatus.name) - .where(context: 'tags') - .where('taggable_id = ci_builds.id').select('1') - - where("EXISTS (?)", matcher) - end - scope :queued_before, ->(time) { where(arel_table[:queued_at].lt(time)) } scope :preload_project_and_pipeline_project, -> do @@ -212,7 +193,7 @@ module Ci acts_as_taggable - add_authentication_token_field :token, encrypted: :optional + add_authentication_token_field :token, encrypted: :required before_save :ensure_token before_destroy { unscoped_project } @@ -344,7 +325,11 @@ module Ci build.run_after_commit do build.run_status_commit_hooks! - BuildFinishedWorker.perform_async(id) + if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project, default_enabled: :yaml) + Ci::BuildFinishedWorker.perform_async(id) + else + ::BuildFinishedWorker.perform_async(id) + end end end @@ -758,6 +743,14 @@ module Ci self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token) end + def tag_list + if tags.loaded? + tags.map(&:name) + else + super + end + end + def has_tags? tag_list.any? end @@ -782,7 +775,7 @@ module Ci return unless project project.execute_hooks(build_data.dup, :job_hooks) if project.has_active_hooks?(:job_hooks) - project.execute_services(build_data.dup, :job_hooks) if project.has_active_services?(:job_hooks) + project.execute_integrations(build_data.dup, :job_hooks) if project.has_active_integrations?(:job_hooks) end def browsable_artifacts? @@ -939,8 +932,7 @@ module Ci end def supports_artifacts_exclude? - options&.dig(:artifacts, :exclude)&.any? && - Gitlab::Ci::Features.artifacts_exclude_enabled? + options&.dig(:artifacts, :exclude)&.any? end def multi_build_steps? diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb index d39e0411a79..c4a04d42a1e 100644 --- a/app/models/ci/build_dependencies.rb +++ b/app/models/ci/build_dependencies.rb @@ -37,12 +37,20 @@ module Ci next [] unless processable.pipeline_id # we don't have any dependency when creating the pipeline deps = model_class.where(pipeline_id: processable.pipeline_id).latest - deps = from_previous_stages(deps) - deps = from_needs(deps) + deps = find_dependencies(processable, deps) + from_dependencies(deps).to_a end end + def find_dependencies(processable, deps) + if processable.scheduling_type_dag? + from_needs(deps) + else + from_previous_stages(deps) + end + end + # Dependencies from the same parent-pipeline hierarchy excluding # the current job's pipeline def cross_pipeline @@ -125,8 +133,6 @@ module Ci end def from_needs(scope) - return scope unless processable.scheduling_type_dag? - needs_names = processable.needs.artifacts.select(:name) scope.where(name: needs_names) end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index f009e4c6aa1..50775f578f0 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -22,8 +22,8 @@ module Ci validates :build, presence: true validates :secrets, json_schema: { filename: 'build_metadata_secrets' } - serialize :config_options, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize - serialize :config_variables, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize + serialize :config_options, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize + serialize :config_variables, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize chronic_duration_attr_reader :timeout_human_readable, :timeout diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 25f4a06088d..3fa9a484b0c 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -14,13 +14,7 @@ module Ci belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id - default_value_for :data_store do - if Feature.enabled?(:dedicated_redis_trace_chunks, type: :ops) - :redis_trace_chunks - else - :redis - end - end + default_value_for :data_store, :redis_trace_chunks after_create { metrics.increment_trace_operation(operation: :chunked) } @@ -115,7 +109,7 @@ module Ci raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0 return if offset == size # Skip the following process as it doesn't affect anything - self.append("", offset) + self.append(+"", offset) end def append(new_data, offset) diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb index fab85fae33d..3bfac2b33c0 100644 --- a/app/models/ci/build_trace_chunks/fog.rb +++ b/app/models/ci/build_trace_chunks/fog.rb @@ -25,14 +25,36 @@ module Ci files.create(create_attributes(model, new_data)) end + # This is the sequence that causes append_data to be called: + # + # 1. Runner sends a PUT /api/v4/jobs/:id to indicate the job is canceled or finished. + # 2. UpdateBuildStateService#accept_build_state! persists all live job logs to object storage (or filesystem). + # 3. UpdateBuildStateService#accept_build_state! returns a 202 to the runner. + # 4. The runner continues to send PATCH requests with job logs until all logs have been sent and received. + # 5. If the last PATCH request arrives after the job log has been persisted, we + # retrieve the data from object storage to append the remaining lines. def append_data(model, new_data, offset) if offset > 0 truncated_data = data(model).to_s.byteslice(0, offset) - new_data = truncated_data + new_data + new_data = append_strings(truncated_data, new_data) end set_data(model, new_data) new_data.bytesize + rescue Encoding::CompatibilityError => e + Gitlab::ErrorTracking.track_and_raise_exception( + e, + build_id: model.build_id, + chunk_index: model.chunk_index, + chunk_start_offset: model.start_offset, + chunk_end_offset: model.end_offset, + chunk_size: model.size, + chunk_data_store: model.data_store, + offset: offset, + old_data_encoding: truncated_data.encoding.to_s, + new_data: new_data, + new_data_size: new_data.bytesize, + new_data_encoding: new_data.encoding.to_s) end def size(model) @@ -57,6 +79,17 @@ module Ci private + def append_strings(old_data, new_data) + if Feature.enabled?(:ci_job_trace_force_encode, default_enabled: :yaml) + # When object storage is in use, old_data may be retrieved in UTF-8. + old_data = old_data.force_encoding(Encoding::ASCII_8BIT) + # new_data should already be in ASCII-8BIT, but just in case it isn't, do this. + new_data = new_data.force_encoding(Encoding::ASCII_8BIT) + end + + old_data + new_data + end + def key(model) key_raw(model.build_id, model.chunk_index) end diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 47b91fcf2ce..e5cb2026503 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -10,6 +10,7 @@ module Ci class Group include StaticModel include Gitlab::Utils::StrongMemoize + include GlobalID::Identification attr_reader :project, :stage, :name, :jobs @@ -22,6 +23,10 @@ module Ci @jobs = jobs end + def id + "#{stage.id}-#{name}" + end + def ==(other) other.present? && other.is_a?(self.class) && project == other.project && diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb index e083caa8751..5aee4c924af 100644 --- a/app/models/ci/instance_variable.rb +++ b/app/models/ci/instance_variable.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class InstanceVariable < ApplicationRecord + class InstanceVariable < ::Ci::BaseModel extend Gitlab::Ci::Model extend Gitlab::ProcessMemoryCache::Helper include Ci::NewHasVariable diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 6a7a2b3f6bd..46c976d5616 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -33,6 +33,7 @@ module Ci secret_detection: 'gl-secret-detection-report.json', dependency_scanning: 'gl-dependency-scanning-report.json', container_scanning: 'gl-container-scanning-report.json', + cluster_image_scanning: 'gl-cluster-image-scanning-report.json', dast: 'gl-dast-report.json', license_scanning: 'gl-license-scanning-report.json', performance: 'performance.json', @@ -71,6 +72,7 @@ module Ci secret_detection: :raw, dependency_scanning: :raw, container_scanning: :raw, + cluster_image_scanning: :raw, dast: :raw, license_scanning: :raw, @@ -108,6 +110,7 @@ module Ci sast secret_detection requirements + cluster_image_scanning ].freeze TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze @@ -181,6 +184,8 @@ module Ci scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) } scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } + scope :for_project, ->(project) { where(project_id: project) } + scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) } delegate :filename, :exists?, :open, to: :file @@ -210,7 +215,8 @@ module Ci coverage_fuzzing: 23, ## EE-specific browser_performance: 24, ## EE-specific load_performance: 25, ## EE-specific - api_fuzzing: 26 ## EE-specific + api_fuzzing: 26, ## EE-specific + cluster_image_scanning: 27 ## EE-specific } # `file_location` indicates where actual files are stored. diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb index 283ad4a190d..99118f8090b 100644 --- a/app/models/ci/job_token/project_scope_link.rb +++ b/app/models/ci/job_token/project_scope_link.rb @@ -19,6 +19,10 @@ module Ci validates :target_project, presence: true validate :not_self_referential_link + def self.for_source_and_target(source_project, target_project) + self.find_by(source_project: source_project, target_project: target_project) + end + private def not_self_referential_link diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index b9a8a44bd6b..0663052f51d 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -7,12 +7,52 @@ module Ci belongs_to :project belongs_to :build, class_name: 'Ci::Build' + scope :ref_protected, -> { where(protected: true) } + scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) } + def self.upsert_from_build!(build) - entry = self.new(build: build, project: build.project, protected: build.protected?) + entry = self.new(args_from_build(build)) entry.validate! self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) end + + def self.args_from_build(build) + args = { + build: build, + project: build.project, + protected: build.protected? + } + + if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) + args.merge(instance_runners_enabled: shareable?(build)) + else + args + end + end + private_class_method :args_from_build + + def self.shareable?(build) + shared_runner_enabled?(build) && + builds_access_level?(build) && + project_not_removed?(build) + end + private_class_method :shareable? + + def self.shared_runner_enabled?(build) + build.project.shared_runners.exists? + end + private_class_method :shared_runner_enabled? + + def self.project_not_removed?(build) + !build.project.pending_delete? + end + private_class_method :project_not_removed? + + def self.builds_access_level?(build) + build.project.project_feature.builds_access_level.nil? || build.project.project_feature.builds_access_level > 0 + end + private_class_method :builds_access_level? end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 159d9d10878..5d079f57267 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -29,6 +29,8 @@ module Ci BridgeStatusError = Class.new(StandardError) + paginates_per 15 + sha_attribute :source_sha sha_attribute :target_sha @@ -222,7 +224,7 @@ module Ci end after_transition [:created, :waiting_for_resource, :preparing, :pending, :running] => :success do |pipeline| - # We wait a little bit to ensure that all BuildFinishedWorkers finish first + # We wait a little bit to ensure that all Ci::BuildFinishedWorkers finish first # because this is where some metrics like code coverage is parsed and stored # in CI build records which the daily build metrics worker relies on. pipeline.run_after_commit { Ci::DailyBuildGroupReportResultsWorker.perform_in(10.minutes, pipeline.id) } @@ -577,11 +579,11 @@ module Ci canceled? && auto_canceled_by_id? end - def cancel_running(retries: nil) + def cancel_running(retries: 1) commit_status_relations = [:project, :pipeline] ci_build_relations = [:deployment, :taggings] - retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables| + retry_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables| cancelables.find_in_batches do |batch| ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations) ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations) @@ -594,7 +596,7 @@ module Ci end end - def auto_cancel_running(pipeline, retries: nil) + def auto_cancel_running(pipeline, retries: 1) update(auto_canceled_by: pipeline) cancel_running(retries: retries) do |job| @@ -610,8 +612,6 @@ module Ci # rubocop: enable CodeReuse/ServiceClass def lazy_ref_commit - return unless ::Gitlab::Ci::Features.pipeline_latest? - BatchLoader.for(ref).batch do |refs, loader| next unless project.repository_exists? @@ -623,11 +623,6 @@ module Ci def latest? return false unless git_ref && commit.present? - - unless ::Gitlab::Ci::Features.pipeline_latest? - return project.commit(git_ref) == commit - end - return false if lazy_ref_commit.nil? lazy_ref_commit.id == commit.id @@ -861,7 +856,7 @@ module Ci def execute_hooks project.execute_hooks(pipeline_data, :pipeline_hooks) if project.has_active_hooks?(:pipeline_hooks) - project.execute_services(pipeline_data, :pipeline_hooks) if project.has_active_services?(:pipeline_hooks) + project.execute_integrations(pipeline_data, :pipeline_hooks) if project.has_active_integrations?(:pipeline_hooks) end # All the merge requests for which the current pipeline runs/ran against @@ -911,7 +906,7 @@ module Ci def same_family_pipeline_ids ::Gitlab::Ci::PipelineObjectHierarchy.new( - self.class.default_scoped.where(id: root_ancestor), options: { same_project: true } + self.class.default_scoped.where(id: root_ancestor), options: { project_condition: :same } ).base_and_descendants.select(:id) end @@ -932,29 +927,34 @@ module Ci Environment.where(id: environment_ids) end - # Without using `unscoped`, caller scope is also included into the query. - # Using `unscoped` here will be redundant after Rails 6.1 + # With multi-project and parent-child pipelines + def self_and_upstreams + object_hierarchy.base_and_ancestors + end + + # With multi-project and parent-child pipelines + def self_with_upstreams_and_downstreams + object_hierarchy.all_objects + end + + # With only parent-child pipelines + def self_and_ancestors + object_hierarchy(project_condition: :same).base_and_ancestors + end + + # With only parent-child pipelines def self_and_descendants - ::Gitlab::Ci::PipelineObjectHierarchy - .new(self.class.unscoped.where(id: id), options: { same_project: true }) - .base_and_descendants + object_hierarchy(project_condition: :same).base_and_descendants end def root_ancestor return self unless child? - Gitlab::Ci::PipelineObjectHierarchy - .new(self.class.unscoped.where(id: id), options: { same_project: true }) + object_hierarchy(project_condition: :same) .base_and_ancestors(hierarchy_order: :desc) .first end - def self_with_ancestors_and_descendants(same_project: false) - ::Gitlab::Ci::PipelineObjectHierarchy - .new(self.class.unscoped.where(id: id), options: { same_project: same_project }) - .all_objects - end - def bridge_triggered? source_bridge.present? end @@ -1026,8 +1026,6 @@ module Ci end def can_generate_codequality_reports? - return false unless ::Gitlab::Ci::Features.display_quality_on_mr_diff?(project) - has_reports?(Ci::JobArtifact.codequality_reports) end @@ -1214,14 +1212,6 @@ module Ci self.ci_ref = Ci::Ref.ensure_for(self) end - def base_and_ancestors(same_project: false) - # Without using `unscoped`, caller scope is also included into the query. - # Using `unscoped` here will be redundant after Rails 6.1 - ::Gitlab::Ci::PipelineObjectHierarchy - .new(self.class.unscoped.where(id: id), options: { same_project: same_project }) - .base_and_ancestors - end - # We need `base_and_ancestors` in a specific order to "break" when needed. # If we use `find_each`, then the order is broken. # rubocop:disable Rails/FindEach @@ -1232,7 +1222,7 @@ module Ci source_bridge.pending! Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass else - base_and_ancestors.includes(:source_bridge).each do |pipeline| + self_and_upstreams.includes(:source_bridge).each do |pipeline| break unless pipeline.bridge_waiting? pipeline.source_bridge.pending! @@ -1315,6 +1305,13 @@ module Ci project.repository.keep_around(self.sha, self.before_sha) end + + # Without using `unscoped`, caller scope is also included into the query. + # Using `unscoped` here will be redundant after Rails 6.1 + def object_hierarchy(options = {}) + ::Gitlab::Ci::PipelineObjectHierarchy + .new(self.class.unscoped.where(id: id), options: options) + end end end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 71110ef0696..a541dca47de 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -13,7 +13,7 @@ module Ci include Gitlab::Utils::StrongMemoize include TaggableQueries - add_authentication_token_field :token, encrypted: -> { Feature.enabled?(:ci_runners_tokens_optional_encryption, default_enabled: true) ? :optional : :required } + add_authentication_token_field :token, encrypted: :optional enum access_level: { not_protected: 0, @@ -214,15 +214,15 @@ module Ci Arel.sql("(#{arel_tag_names_array.to_sql})") ] - # we use distinct to de-duplicate data - distinct.pluck(*unique_params).map do |values| + group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| Gitlab::Ci::Matching::RunnerMatcher.new({ - runner_type: values[0], - public_projects_minutes_cost_factor: values[1], - private_projects_minutes_cost_factor: values[2], - run_untagged: values[3], - access_level: values[4], - tag_list: values[5] + runner_ids: values[0], + runner_type: values[1], + public_projects_minutes_cost_factor: values[2], + private_projects_minutes_cost_factor: values[3], + run_untagged: values[4], + access_level: values[5], + tag_list: values[6] }) end end @@ -230,6 +230,7 @@ module Ci def runner_matcher strong_memoize(:runner_matcher) do Gitlab::Ci::Matching::RunnerMatcher.new({ + runner_ids: [id], runner_type: runner_type, public_projects_minutes_cost_factor: public_projects_minutes_cost_factor, private_projects_minutes_cost_factor: private_projects_minutes_cost_factor, |