diff options
Diffstat (limited to 'app/services/ci')
15 files changed, 156 insertions, 176 deletions
diff --git a/app/services/ci/build_report_result_service.rb b/app/services/ci/build_report_result_service.rb index 8bdb51320f9..f9146b3677a 100644 --- a/app/services/ci/build_report_result_service.rb +++ b/app/services/ci/build_report_result_service.rb @@ -22,7 +22,7 @@ module Ci private def generate_test_suite_report(build) - build.collect_test_reports!(Gitlab::Ci::Reports::TestReports.new) + build.collect_test_reports!(Gitlab::Ci::Reports::TestReport.new) end def tests_params(test_suite) diff --git a/app/services/ci/external_pull_requests/create_pipeline_service.rb b/app/services/ci/external_pull_requests/create_pipeline_service.rb index 66127c94d35..ffc129eccda 100644 --- a/app/services/ci/external_pull_requests/create_pipeline_service.rb +++ b/app/services/ci/external_pull_requests/create_pipeline_service.rb @@ -10,24 +10,12 @@ module Ci return pull_request_not_open_error unless pull_request.open? return pull_request_branch_error unless pull_request.actual_branch_head? - create_pipeline_for(pull_request) - end - - private - - def create_pipeline_for(pull_request) Ci::ExternalPullRequests::CreatePipelineWorker.perform_async( project.id, current_user.id, pull_request.id ) end - def create_params(pull_request) - { - ref: pull_request.source_ref, - source_sha: pull_request.source_sha, - target_sha: pull_request.target_sha - } - end + private def pull_request_not_open_error ServiceResponse.error(message: 'The pull request is not opened', payload: nil) diff --git a/app/services/ci/generate_coverage_reports_service.rb b/app/services/ci/generate_coverage_reports_service.rb index 12b1f19f4b5..81f26e84ef8 100644 --- a/app/services/ci/generate_coverage_reports_service.rb +++ b/app/services/ci/generate_coverage_reports_service.rb @@ -32,5 +32,18 @@ module Ci def latest?(base_pipeline, head_pipeline, data) data&.fetch(:key, nil) == key(base_pipeline, head_pipeline) end + + private + + def key(base_pipeline, head_pipeline) + [ + base_pipeline&.id, last_update_timestamp(base_pipeline), + head_pipeline&.id, last_update_timestamp(head_pipeline) + ] + end + + def last_update_timestamp(pipeline_hierarchy) + pipeline_hierarchy&.self_and_descendants&.maximum(:updated_at) + end end end diff --git a/app/services/ci/job_artifacts/create_service.rb b/app/services/ci/job_artifacts/create_service.rb index 635111130d6..05f8e804c67 100644 --- a/app/services/ci/job_artifacts/create_service.rb +++ b/app/services/ci/job_artifacts/create_service.rb @@ -5,10 +5,7 @@ module Ci class CreateService < ::BaseService include Gitlab::Utils::UsageData - ArtifactsExistError = Class.new(StandardError) - LSIF_ARTIFACT_TYPE = 'lsif' - METRICS_REPORT_UPLOAD_EVENT_NAME = 'i_testing_metrics_report_artifact_uploaders' OBJECT_STORAGE_ERRORS = [ Errno::EIO, @@ -74,10 +71,6 @@ module Ci Ci::JobArtifact.max_artifact_size(type: type, project: project) end - def forbidden_type_error(type) - error("#{type} artifacts are forbidden", :forbidden) - end - def too_large_error error('file size has reached maximum size limit', :payload_too_large) end @@ -160,10 +153,8 @@ module Ci ) end - def track_artifact_uploader(artifact) - return unless artifact.file_type == 'metrics' - - track_usage_event(METRICS_REPORT_UPLOAD_EVENT_NAME, @job.user_id) + def track_artifact_uploader(_artifact) + # Overridden in EE end def parse_dotenv_artifact(artifact) @@ -172,3 +163,5 @@ module Ci end end end + +Ci::JobArtifacts::CreateService.prepend_mod diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb index 49b65f13804..9d6b413ce59 100644 --- a/app/services/ci/job_artifacts/destroy_batch_service.rb +++ b/app/services/ci/job_artifacts/destroy_batch_service.rb @@ -184,10 +184,12 @@ module Ci project_ids << artifact.project_id end - Gitlab::ProjectStatsRefreshConflictsLogger.warn_skipped_artifact_deletion_during_stats_refresh( - method: 'Ci::JobArtifacts::DestroyBatchService#execute', - project_ids: project_ids - ) + if project_ids.any? + Gitlab::ProjectStatsRefreshConflictsLogger.warn_skipped_artifact_deletion_during_stats_refresh( + method: 'Ci::JobArtifacts::DestroyBatchService#execute', + project_ids: project_ids + ) + end end end end diff --git a/app/services/ci/pipeline_artifacts/coverage_report_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb index b0acb1d5a0b..c11a8f7a0fd 100644 --- a/app/services/ci/pipeline_artifacts/coverage_report_service.rb +++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb @@ -9,17 +9,11 @@ module Ci end def execute - return if pipeline.has_coverage_reports? return if report.empty? - pipeline.pipeline_artifacts.create!( - project_id: pipeline.project_id, - file_type: :code_coverage, - file_format: Ci::PipelineArtifact::REPORT_TYPES.fetch(:code_coverage), - size: carrierwave_file["tempfile"].size, - file: carrierwave_file, - expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now - ) + Ci::PipelineArtifact.create_or_replace_for_pipeline!(**pipeline_artifact_params).tap do |pipeline_artifact| + Gitlab::AppLogger.info(log_params(pipeline_artifact)) + end end private @@ -32,6 +26,15 @@ module Ci end end + def pipeline_artifact_params + { + pipeline: pipeline, + file_type: :code_coverage, + file: carrierwave_file, + size: carrierwave_file['tempfile'].size + } + end + def carrierwave_file strong_memoize(:carrier_wave_file) do CarrierWaveStringFile.new_file( @@ -41,6 +44,15 @@ module Ci ) end end + + def log_params(pipeline_artifact) + { + project_id: pipeline.project_id, + pipeline_id: pipeline.id, + pipeline_artifact_id: pipeline_artifact.id, + message: "Created code coverage for pipeline." + } + end end end end diff --git a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb index 4d1b2e07d7f..676c2ecb257 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb @@ -78,7 +78,7 @@ module Ci def status_for_array(statuses, dag:) result = Gitlab::Ci::Status::Composite - .new(statuses, dag: dag, project: pipeline.project) + .new(statuses, dag: dag) .status result || 'success' end diff --git a/app/services/ci/queue/build_queue_service.rb b/app/services/ci/queue/build_queue_service.rb index fefbdb151ec..2deebc1d725 100644 --- a/app/services/ci/queue/build_queue_service.rb +++ b/app/services/ci/queue/build_queue_service.rb @@ -24,25 +24,7 @@ module Ci # rubocop:disable CodeReuse/ActiveRecord def builds_for_group_runner - if strategy.use_denormalized_data_strategy? - strategy.builds_for_group_runner - else - # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL` - groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces) - - hierarchy_groups = Gitlab::ObjectHierarchy - .new(groups) - .base_and_descendants - - projects = Project.where(namespace_id: hierarchy_groups) - .with_group_runners_enabled - .with_builds_enabled - .without_deleted - - relation = new_builds.where(project: projects) - - order(relation) - end + strategy.builds_for_group_runner end def builds_for_project_runner @@ -80,11 +62,7 @@ module Ci def strategy strong_memoize(:strategy) do - if ::Feature.enabled?(:ci_pending_builds_queue_source, runner) - Queue::PendingBuildsStrategy.new(runner) - else - Queue::BuildsTableStrategy.new(runner) - end + Queue::PendingBuildsStrategy.new(runner) end end diff --git a/app/services/ci/queue/builds_table_strategy.rb b/app/services/ci/queue/builds_table_strategy.rb deleted file mode 100644 index c27c10bd18d..00000000000 --- a/app/services/ci/queue/builds_table_strategy.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -module Ci - module Queue - class BuildsTableStrategy - attr_reader :runner - - def initialize(runner) - @runner = runner - end - - # rubocop:disable CodeReuse/ActiveRecord - def builds_for_shared_runner - relation = new_builds - # don't run projects which have not enabled shared runners and builds - .joins('INNER JOIN projects ON ci_builds.project_id = projects.id') - .where(projects: { shared_runners_enabled: true, pending_delete: false }) - .joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id') - .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') - - if Feature.enabled?(:ci_queueing_disaster_recovery_disable_fair_scheduling, runner, type: :ops) - # if disaster recovery is enabled, we fallback to FIFO scheduling - relation.order('ci_builds.id ASC') - else - # Implement fair scheduling - # this returns builds that are ordered by number of running builds - # we prefer projects that don't use shared runners at all - relation - .joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id = project_builds.project_id") - .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_builds.id ASC') - end - end - - def builds_for_group_runner - raise NotImplementedError - end - - def builds_matching_tag_ids(relation, ids) - # pick builds that does not have other tags than runner's one - relation.matches_tag_ids(ids) - end - - def builds_with_any_tags(relation) - # pick builds that have at least one tag - relation.with_any_tags - end - - def order(relation) - relation.order('id ASC') - end - - def new_builds - ::Ci::Build.pending.unstarted - end - - def build_ids(relation) - relation.pluck(:id) - end - - def use_denormalized_data_strategy? - false - end - - private - - def running_builds_for_shared_runners - ::Ci::Build.running - .where(runner: ::Ci::Runner.instance_type) - .group(:project_id) - .select(:project_id, 'COUNT(*) AS running_builds') - end - # rubocop:enable CodeReuse/ActiveRecord - end - end -end diff --git a/app/services/ci/queue/pending_builds_strategy.rb b/app/services/ci/queue/pending_builds_strategy.rb index f2eba0681db..c8bdbba5e65 100644 --- a/app/services/ci/queue/pending_builds_strategy.rb +++ b/app/services/ci/queue/pending_builds_strategy.rb @@ -23,19 +23,11 @@ module Ci end def builds_matching_tag_ids(relation, ids) - if use_denormalized_data_strategy? - relation.for_tags(runner.tags_ids) - else - relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id')) - end + relation.for_tags(runner.tags_ids) end def builds_with_any_tags(relation) - if use_denormalized_data_strategy? - relation.where('cardinality(tag_ids) > 0') - else - relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id')) - end + relation.where('cardinality(tag_ids) > 0') end def order(relation) @@ -50,23 +42,10 @@ module Ci relation.pluck(:build_id) end - def use_denormalized_data_strategy? - ::Feature.enabled?(:ci_queuing_use_denormalized_data_strategy) - end - private def builds_available_for_shared_runners - if use_denormalized_data_strategy? - new_builds.with_instance_runners - else - new_builds - # don't run projects which have not enabled shared runners and builds - .joins('INNER JOIN projects ON ci_pending_builds.project_id = projects.id') - .where(projects: { shared_runners_enabled: true, pending_delete: false }) - .joins('LEFT JOIN project_features ON ci_pending_builds.project_id = project_features.project_id') - .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') - end + new_builds.with_instance_runners end def builds_ordered_for_shared_runners(relation) diff --git a/app/services/ci/runners/reconcile_existing_runner_versions_service.rb b/app/services/ci/runners/reconcile_existing_runner_versions_service.rb new file mode 100644 index 00000000000..e04079bfe27 --- /dev/null +++ b/app/services/ci/runners/reconcile_existing_runner_versions_service.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Ci + module Runners + class ReconcileExistingRunnerVersionsService + include BaseServiceUtility + + VERSION_BATCH_SIZE = 100 + + def execute + insert_result = insert_runner_versions + total_deleted = cleanup_runner_versions(insert_result[:versions_from_runners]) + total_updated = update_status_on_outdated_runner_versions(insert_result[:versions_from_runners]) + + success({ + total_inserted: insert_result[:new_record_count], + total_updated: total_updated, + total_deleted: total_deleted + }) + end + + private + + def upgrade_check + Gitlab::Ci::RunnerUpgradeCheck.instance + end + + # rubocop: disable CodeReuse/ActiveRecord + def insert_runner_versions + versions_from_runners = Set[] + new_record_count = 0 + Ci::Runner.distinct_each_batch(column: :version, of: VERSION_BATCH_SIZE) do |version_batch| + batch_versions = version_batch.pluck(:version).to_set + versions_from_runners += batch_versions + + # Avoid hitting primary DB + already_existing_versions = Ci::RunnerVersion.where(version: batch_versions).pluck(:version) + new_versions = batch_versions - already_existing_versions + + if new_versions.any? + new_record_count += Ci::RunnerVersion.insert_all( + new_versions.map { |v| { version: v } }, + returning: :version, + unique_by: :version).count + end + end + + { versions_from_runners: versions_from_runners, new_record_count: new_record_count } + end + + def cleanup_runner_versions(versions_from_runners) + Ci::RunnerVersion.where.not(version: versions_from_runners).delete_all + end + # rubocop: enable CodeReuse/ActiveRecord + + def outdated_runner_versions + Ci::RunnerVersion.potentially_outdated + end + + def update_status_on_outdated_runner_versions(versions_from_runners) + total_updated = 0 + + outdated_runner_versions.each_batch(of: VERSION_BATCH_SIZE) do |version_batch| + updated = version_batch + .select { |runner_version| versions_from_runners.include?(runner_version['version']) } + .filter_map { |runner_version| runner_version_with_updated_status(runner_version) } + + if updated.any? + total_updated += Ci::RunnerVersion.upsert_all(updated, unique_by: :version).count + end + end + + total_updated + end + + def runner_version_with_updated_status(runner_version) + version = runner_version['version'] + suggestion = upgrade_check.check_runner_upgrade_status(version) + new_status = suggestion.each_key.first + + if new_status != :error && new_status != runner_version['status'].to_sym + { + version: version, + status: Ci::RunnerVersion.statuses[new_status] + } + end + end + end + end +end diff --git a/app/services/ci/runners/register_runner_service.rb b/app/services/ci/runners/register_runner_service.rb index 196d2de1a65..6588cd7e248 100644 --- a/app/services/ci/runners/register_runner_service.rb +++ b/app/services/ci/runners/register_runner_service.rb @@ -8,7 +8,19 @@ module Ci return unless runner_type_attrs - ::Ci::Runner.create(attributes.merge(runner_type_attrs)) + runner = ::Ci::Runner.new(attributes.merge(runner_type_attrs)) + + Ci::BulkInsertableTags.with_bulk_insert_tags do + Ci::Runner.transaction do + if runner.save + Gitlab::Ci::Tags::BulkInsert.bulk_insert_tags!([runner]) + else + raise ActiveRecord::Rollback + end + end + end + + runner end private diff --git a/app/services/ci/test_failure_history_service.rb b/app/services/ci/test_failure_history_service.rb index 7323ad417ea..2214a6a2729 100644 --- a/app/services/ci/test_failure_history_service.rb +++ b/app/services/ci/test_failure_history_service.rb @@ -81,7 +81,7 @@ module Ci def generate_test_suite!(build) # Returns an instance of Gitlab::Ci::Reports::TestSuite - build.collect_test_reports!(Gitlab::Ci::Reports::TestReports.new) + build.collect_test_reports!(Gitlab::Ci::Reports::TestReport.new) end def ci_unit_test_attrs(batch) diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb index a525ea179e0..58927a90b6e 100644 --- a/app/services/ci/update_build_queue_service.rb +++ b/app/services/ci/update_build_queue_service.rb @@ -14,8 +14,6 @@ module Ci # Add a build to the pending builds queue # def push(build, transition) - return unless maintain_pending_builds_queue? - raise InvalidQueueTransition unless transition.to == 'pending' transition.within_transaction do @@ -33,8 +31,6 @@ module Ci # Remove a build from the pending builds queue # def pop(build, transition) - return unless maintain_pending_builds_queue? - raise InvalidQueueTransition unless transition.from == 'pending' transition.within_transaction { remove!(build) } @@ -57,7 +53,6 @@ module Ci # Add shared runner build tracking entry (used for queuing). # def track(build, transition) - return unless maintain_pending_builds_queue? return unless build.shared_runner_build? raise InvalidQueueTransition unless transition.to == 'running' @@ -78,7 +73,6 @@ module Ci # queuing). # def untrack(build, transition) - return unless maintain_pending_builds_queue? return unless build.shared_runner_build? raise InvalidQueueTransition unless transition.from == 'running' @@ -115,9 +109,5 @@ module Ci runner.pick_build!(build) end end - - def maintain_pending_builds_queue? - ::Ci::PendingBuild.maintain_denormalized_data? - end end end diff --git a/app/services/ci/update_pending_build_service.rb b/app/services/ci/update_pending_build_service.rb index 733b684bcc6..2118dbcc19e 100644 --- a/app/services/ci/update_pending_build_service.rb +++ b/app/services/ci/update_pending_build_service.rb @@ -15,8 +15,6 @@ module Ci end def execute - return unless ::Ci::PendingBuild.maintain_denormalized_data? - @model.pending_builds.each_batch do |relation| relation.update_all(@update_params) end |