diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
commit | edaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch) | |
tree | 11f143effbfeba52329fb7afbd05e6e2a3790241 /app/services | |
parent | d8a5691316400a0f7ec4f83832698f1988eb27c1 (diff) | |
download | gitlab-ce-edaa33dee2ff2f7ea3fac488d41558eb5f86d68c.tar.gz |
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'app/services')
75 files changed, 947 insertions, 478 deletions
diff --git a/app/services/alert_management/alerts/update_service.rb b/app/services/alert_management/alerts/update_service.rb index 089715a42fb..7a9bcf2a52d 100644 --- a/app/services/alert_management/alerts/update_service.rb +++ b/app/services/alert_management/alerts/update_service.rb @@ -2,7 +2,7 @@ module AlertManagement module Alerts - class UpdateService + class UpdateService < ::BaseProjectService include Gitlab::Utils::StrongMemoize # @param alert [AlertManagement::Alert] @@ -10,10 +10,10 @@ module AlertManagement # @param params [Hash] Attributes of the alert def initialize(alert, current_user, params) @alert = alert - @current_user = current_user - @params = params @param_errors = [] @status = params.delete(:status) + + super(project: alert.project, current_user: current_user, params: params) end def execute @@ -36,7 +36,7 @@ module AlertManagement private - attr_reader :alert, :current_user, :params, :param_errors, :status + attr_reader :alert, :param_errors, :status def allowed? current_user&.can?(:update_alert_management_alert, alert) @@ -109,7 +109,7 @@ module AlertManagement end def add_assignee_system_note(old_assignees) - SystemNoteService.change_issuable_assignees(alert, alert.project, current_user, old_assignees) + SystemNoteService.change_issuable_assignees(alert, project, current_user, old_assignees) end # ------ Status-related behavior ------- @@ -129,6 +129,7 @@ module AlertManagement def handle_status_change add_status_change_system_note resolve_todos if alert.resolved? + sync_to_incident if should_sync_to_incident? end def add_status_change_system_note @@ -139,6 +140,22 @@ module AlertManagement todo_service.resolve_todos_for_target(alert, current_user) end + def sync_to_incident + ::Issues::UpdateService.new( + project: project, + current_user: current_user, + params: { escalation_status: { status: status } } + ).execute(alert.issue) + end + + def should_sync_to_incident? + Feature.enabled?(:incident_escalations, project) && + alert.issue && + alert.issue.supports_escalation? && + alert.issue.escalation_status && + alert.issue.escalation_status.status != alert.status + end + def filter_duplicate # Only need to check if changing to an open status return unless params[:status_event] && AlertManagement::Alert.open_status?(status) @@ -154,7 +171,7 @@ module AlertManagement def open_alerts strong_memoize(:open_alerts) do - AlertManagement::Alert.for_fingerprint(alert.project, alert.fingerprint).open + AlertManagement::Alert.for_fingerprint(project, alert.fingerprint).open end end @@ -166,7 +183,7 @@ module AlertManagement def open_alert_url_params open_alert = open_alerts.first - alert_path = Gitlab::Routing.url_helpers.details_project_alert_management_path(alert.project, open_alert) + alert_path = Gitlab::Routing.url_helpers.details_project_alert_management_path(project, open_alert) { link_start: '<a href="%{url}">'.html_safe % { url: alert_path }, diff --git a/app/services/audit_event_service.rb b/app/services/audit_event_service.rb index 1426bf25a00..f2b1d89161c 100644 --- a/app/services/audit_event_service.rb +++ b/app/services/audit_event_service.rb @@ -141,7 +141,7 @@ class AuditEventService event.save! if should_save_database?(@save_type) stream_event_to_external_destinations(event) if should_save_stream?(@save_type) rescue StandardError => e - Gitlab::ErrorTracking.track_exception(e, audit_event_type: event.class.to_s) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, audit_event_type: event.class.to_s) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index ea4723c9e28..a92a2c8aef5 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -124,7 +124,8 @@ module Auth type: type, name: path.to_s, actions: authorized_actions, - migration_eligible: self.class.migration_eligible(project: requested_project) + migration_eligible: self.class.migration_eligible(project: requested_project), + cdn_redirect: cdn_redirect }.compact end @@ -145,6 +146,16 @@ module Auth # we'll remove them manually from this deny list, and their new repositories will become eligible. Feature.disabled?(:container_registry_migration_phase1_deny, project.root_ancestor) && Feature.enabled?(:container_registry_migration_phase1_allow, project) + rescue ContainerRegistry::Path::InvalidRegistryPathError => ex + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(ex, **Gitlab::ApplicationContext.current) + false + end + + # This is used to determine whether blob download requests using a given JWT token should be redirected to Google + # Cloud CDN or not. The intent is to enable a percentage of time rollout for this new feature on the Container + # Registry side. See https://gitlab.com/gitlab-org/gitlab/-/issues/349417 for more details. + def cdn_redirect + Feature.enabled?(:container_registry_cdn_redirect) || nil end ## diff --git a/app/services/auto_merge/base_service.rb b/app/services/auto_merge/base_service.rb index da80211f9bb..4ed4368d3b7 100644 --- a/app/services/auto_merge/base_service.rb +++ b/app/services/auto_merge/base_service.rb @@ -6,7 +6,7 @@ module AutoMerge include MergeRequests::AssignsMergeParams def execute(merge_request) - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases + ApplicationRecord.transaction do register_auto_merge_parameters!(merge_request) yield if block_given? end @@ -29,7 +29,7 @@ module AutoMerge end def cancel(merge_request) - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases + ApplicationRecord.transaction do clear_auto_merge_parameters!(merge_request) yield if block_given? end @@ -41,7 +41,7 @@ module AutoMerge end def abort(merge_request, reason) - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases + ApplicationRecord.transaction do clear_auto_merge_parameters!(merge_request) yield if block_given? end diff --git a/app/services/bulk_imports/archive_extraction_service.rb b/app/services/bulk_imports/archive_extraction_service.rb index 9fc828b8e34..caa40d98a76 100644 --- a/app/services/bulk_imports/archive_extraction_service.rb +++ b/app/services/bulk_imports/archive_extraction_service.rb @@ -28,8 +28,8 @@ module BulkImports end def execute - validate_filepath validate_tmpdir + validate_filepath validate_symlink extract_archive @@ -46,7 +46,7 @@ module BulkImports end def validate_tmpdir - raise(BulkImports::Error, 'Invalid target directory') unless File.expand_path(tmpdir).start_with?(Dir.tmpdir) + Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir]) end def validate_symlink diff --git a/app/services/bulk_imports/file_decompression_service.rb b/app/services/bulk_imports/file_decompression_service.rb index fe9017377ec..b76746b199f 100644 --- a/app/services/bulk_imports/file_decompression_service.rb +++ b/app/services/bulk_imports/file_decompression_service.rb @@ -1,21 +1,26 @@ # frozen_string_literal: true +# File Decompression Service allows gzipped files decompression into tmp directory. +# +# @param tmpdir [String] Temp directory to store downloaded file to. Must be located under `Dir.tmpdir`. +# @param filename [String] Name of the file to decompress. module BulkImports class FileDecompressionService include Gitlab::ImportExport::CommandLineUtil ServiceError = Class.new(StandardError) - def initialize(dir:, filename:) - @dir = dir + def initialize(tmpdir:, filename:) + @tmpdir = tmpdir @filename = filename - @filepath = File.join(@dir, @filename) + @filepath = File.join(@tmpdir, @filename) @decompressed_filename = File.basename(@filename, '.gz') - @decompressed_filepath = File.join(@dir, @decompressed_filename) + @decompressed_filepath = File.join(@tmpdir, @decompressed_filename) end def execute - validate_dir + validate_tmpdir + validate_filepath validate_decompressed_file_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) validate_symlink(filepath) @@ -33,10 +38,14 @@ module BulkImports private - attr_reader :dir, :filename, :filepath, :decompressed_filename, :decompressed_filepath + attr_reader :tmpdir, :filename, :filepath, :decompressed_filename, :decompressed_filepath - def validate_dir - raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir) + def validate_filepath + Gitlab::Utils.check_path_traversal!(filepath) + end + + def validate_tmpdir + Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir]) end def validate_decompressed_file_size @@ -48,7 +57,7 @@ module BulkImports end def decompress_file - gunzip(dir: dir, filename: filename) + gunzip(dir: tmpdir, filename: filename) end def size_validator diff --git a/app/services/bulk_imports/file_download_service.rb b/app/services/bulk_imports/file_download_service.rb index d08dc72e30b..8d6ba54cd50 100644 --- a/app/services/bulk_imports/file_download_service.rb +++ b/app/services/bulk_imports/file_download_service.rb @@ -1,6 +1,13 @@ # frozen_string_literal: true -# Downloads a remote file. If no filename is given, it'll use the remote filename +# File Download Service allows remote file download into tmp directory. +# +# @param configuration [BulkImports::Configuration] Config object containing url and access token +# @param relative_url [String] Relative URL to download the file from +# @param tmpdir [String] Temp directory to store downloaded file to. Must be located under `Dir.tmpdir`. +# @param file_size_limit [Integer] Maximum allowed file size +# @param allowed_content_types [Array<String>] Allowed file content types +# @param filename [String] Name of the file to download, if known. Use remote filename if none given. module BulkImports class FileDownloadService ServiceError = Class.new(StandardError) @@ -13,20 +20,21 @@ module BulkImports def initialize( configuration:, relative_url:, - dir:, + tmpdir:, file_size_limit: DEFAULT_FILE_SIZE_LIMIT, allowed_content_types: DEFAULT_ALLOWED_CONTENT_TYPES, filename: nil) @configuration = configuration @relative_url = relative_url @filename = filename - @dir = dir + @tmpdir = tmpdir @file_size_limit = file_size_limit @allowed_content_types = allowed_content_types end def execute - validate_dir + validate_tmpdir + validate_filepath validate_url validate_content_type validate_content_length @@ -40,7 +48,7 @@ module BulkImports private - attr_reader :configuration, :relative_url, :dir, :file_size_limit, :allowed_content_types + attr_reader :configuration, :relative_url, :tmpdir, :file_size_limit, :allowed_content_types def download_file File.open(filepath, 'wb') do |file| @@ -76,8 +84,12 @@ module BulkImports @headers ||= http_client.head(relative_url).headers end - def validate_dir - raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir) + def validate_filepath + Gitlab::Utils.check_path_traversal!(filepath) + end + + def validate_tmpdir + Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir]) end def validate_symlink @@ -119,7 +131,7 @@ module BulkImports end def filepath - @filepath ||= File.join(@dir, filename) + @filepath ||= File.join(@tmpdir, filename) end def filename diff --git a/app/services/bulk_imports/file_export_service.rb b/app/services/bulk_imports/file_export_service.rb index a7e0f998666..a9d06d84277 100644 --- a/app/services/bulk_imports/file_export_service.rb +++ b/app/services/bulk_imports/file_export_service.rb @@ -26,8 +26,10 @@ module BulkImports def export_service case relation - when FileTransfer::ProjectConfig::UPLOADS_RELATION + when FileTransfer::BaseConfig::UPLOADS_RELATION UploadsExportService.new(portable, export_path) + when FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION + LfsObjectsExportService.new(portable, export_path) else raise BulkImports::Error, 'Unsupported relation export type' end diff --git a/app/services/bulk_imports/lfs_objects_export_service.rb b/app/services/bulk_imports/lfs_objects_export_service.rb new file mode 100644 index 00000000000..fa606e4e5a3 --- /dev/null +++ b/app/services/bulk_imports/lfs_objects_export_service.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module BulkImports + class LfsObjectsExportService + include Gitlab::ImportExport::CommandLineUtil + + BATCH_SIZE = 100 + + def initialize(portable, export_path) + @portable = portable + @export_path = export_path + @lfs_json = {} + end + + def execute + portable.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| # rubocop: disable CodeReuse/ActiveRecord + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) + end + + write_lfs_json + end + + private + + attr_reader :portable, :export_path, :lfs_json + + def save_lfs_object(lfs_object) + destination_filepath = File.join(export_path, lfs_object.oid) + + if lfs_object.local_store? + copy_files(lfs_object.file.path, destination_filepath) + else + download(lfs_object.file.url, destination_filepath) + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: portable, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def write_lfs_json + filepath = File.join(export_path, "#{BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION}.json") + + File.write(filepath, lfs_json.to_json) + end + end +end diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index 17cac38ace2..7b1d2207460 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -27,6 +27,10 @@ module Ci job.trace.archive! job.remove_pending_state! + if Feature.enabled?(:datadog_integration_logs_collection, job.project) && job.job_artifacts_trace.present? + job.project.execute_integrations(Gitlab::DataBuilder::ArchiveTrace.build(job), :archive_trace_hooks) + end + # TODO: Remove this logging once we confirmed new live trace architecture is functional. # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. unless job.has_archived_trace? diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index c1f35afba40..d53e136effb 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -95,7 +95,10 @@ module Ci .build! if pipeline.persisted? - schedule_head_pipeline_update + Gitlab::EventStore.publish( + Ci::PipelineCreatedEvent.new(data: { pipeline_id: pipeline.id }) + ) + create_namespace_onboarding_action else # If pipeline is not persisted, try to recover IID @@ -134,12 +137,6 @@ module Ci commit.try(:id) end - def schedule_head_pipeline_update - pipeline.all_merge_requests.opened.each do |merge_request| - UpdateHeadPipelineForMergeRequestWorker.perform_async(merge_request.id) - end - end - def create_namespace_onboarding_action Namespaces::OnboardingPipelineCreatedWorker.perform_async(project.namespace_id) end diff --git a/app/services/ci/destroy_pipeline_service.rb b/app/services/ci/destroy_pipeline_service.rb index 6fbde5d291c..d85e52e1312 100644 --- a/app/services/ci/destroy_pipeline_service.rb +++ b/app/services/ci/destroy_pipeline_service.rb @@ -9,12 +9,12 @@ module Ci pipeline.cancel_running if pipeline.cancelable? - # Ci::Pipeline#destroy triggers `use_fast_destroy :job_artifacts` and - # ci_builds has ON DELETE CASCADE to ci_pipelines. The pipeline, the builds, - # job and pipeline artifacts all get destroyed here. - ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/345664') do - pipeline.reset.destroy! - end + # The pipeline, the builds, job and pipeline artifacts all get destroyed here. + # Ci::Pipeline#destroy triggers fast destroy on job_artifacts and + # build_trace_chunks to remove the records and data stored in object storage. + # ci_builds records are deleted using ON DELETE CASCADE from ci_pipelines + # + pipeline.reset.destroy! ServiceResponse.success(message: 'Pipeline not found') rescue ActiveRecord::RecordNotFound diff --git a/app/services/ci/job_artifacts/delete_project_artifacts_service.rb b/app/services/ci/job_artifacts/delete_project_artifacts_service.rb new file mode 100644 index 00000000000..61394573748 --- /dev/null +++ b/app/services/ci/job_artifacts/delete_project_artifacts_service.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Ci + module JobArtifacts + class DeleteProjectArtifactsService < BaseProjectService + def execute + ExpireProjectBuildArtifactsWorker.perform_async(project.id) + end + end + end +end diff --git a/app/services/ci/job_artifacts/destroy_all_expired_service.rb b/app/services/ci/job_artifacts/destroy_all_expired_service.rb index 7fa56677a0c..c089567ec14 100644 --- a/app/services/ci/job_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/job_artifacts/destroy_all_expired_service.rb @@ -8,13 +8,15 @@ module Ci BATCH_SIZE = 100 LOOP_TIMEOUT = 5.minutes - LOOP_LIMIT = 1000 + SMALL_LOOP_LIMIT = 100 + LARGE_LOOP_LIMIT = 500 EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock' LOCK_TIMEOUT = 6.minutes def initialize @removed_artifacts_count = 0 @start_at = Time.current + @loop_limit = ::Feature.enabled?(:ci_artifact_fast_removal_large_loop_limit, default_enabled: :yaml) ? LARGE_LOOP_LIMIT : SMALL_LOOP_LIMIT end ## @@ -24,6 +26,8 @@ module Ci # preventing multiple `ExpireBuildArtifactsWorker` CRON jobs run concurrently, # which is scheduled every 7 minutes. def execute + return 0 unless ::Feature.enabled?(:ci_destroy_all_expired_service, default_enabled: :yaml) + in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do if ::Feature.enabled?(:ci_destroy_unlocked_job_artifacts) destroy_unlocked_job_artifacts @@ -38,34 +42,13 @@ module Ci private def destroy_unlocked_job_artifacts - loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + loop_until(timeout: LOOP_TIMEOUT, limit: @loop_limit) do artifacts = Ci::JobArtifact.expired_before(@start_at).artifact_unlocked.limit(BATCH_SIZE) service_response = destroy_batch(artifacts) @removed_artifacts_count += service_response[:destroyed_artifacts_count] - - update_locked_status_on_unknown_artifacts if service_response[:destroyed_artifacts_count] == 0 - - # Return a truthy value here to prevent exiting #loop_until - @removed_artifacts_count end end - def update_locked_status_on_unknown_artifacts - build_ids = Ci::JobArtifact.expired_before(@start_at).artifact_unknown.limit(BATCH_SIZE).distinct_job_ids - - return unless build_ids.present? - - locked_pipeline_build_ids = ::Ci::Build.with_pipeline_locked_artifacts.id_in(build_ids).pluck_primary_key - unlocked_pipeline_build_ids = build_ids - locked_pipeline_build_ids - - update_unknown_artifacts(locked_pipeline_build_ids, Ci::JobArtifact.lockeds[:artifacts_locked]) - update_unknown_artifacts(unlocked_pipeline_build_ids, Ci::JobArtifact.lockeds[:unlocked]) - end - - def update_unknown_artifacts(build_ids, locked_value) - Ci::JobArtifact.for_job_ids(build_ids).update_all(locked: locked_value) if build_ids.any? - end - def destroy_job_artifacts_with_slow_iteration Ci::JobArtifact.expired_before(@start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index| # For performance reasons, join with ci_pipelines after the batch is queried. @@ -76,7 +59,7 @@ module Ci @removed_artifacts_count += service_response[:destroyed_artifacts_count] break if loop_timeout? - break if index >= LOOP_LIMIT + break if index >= @loop_limit end end diff --git a/app/services/ci/job_artifacts/expire_project_build_artifacts_service.rb b/app/services/ci/job_artifacts/expire_project_build_artifacts_service.rb new file mode 100644 index 00000000000..836b1d39736 --- /dev/null +++ b/app/services/ci/job_artifacts/expire_project_build_artifacts_service.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Ci + module JobArtifacts + class ExpireProjectBuildArtifactsService + BATCH_SIZE = 1000 + + def initialize(project_id, expiry_time) + @project_id = project_id + @expiry_time = expiry_time + end + + # rubocop:disable CodeReuse/ActiveRecord + def execute + scope = Ci::JobArtifact.for_project(project_id).order(:id) + file_type_values = Ci::JobArtifact.erasable_file_types.map { |file_type| [Ci::JobArtifact.file_types[file_type]] } + from_sql = Arel::Nodes::Grouping.new(Arel::Nodes::ValuesList.new(file_type_values)).as('file_types (file_type)').to_sql + array_scope = Ci::JobArtifact.from(from_sql).select(:file_type) + array_mapping_scope = -> (file_type_expression) { Ci::JobArtifact.where(Ci::JobArtifact.arel_table[:file_type].eq(file_type_expression)) } + + Gitlab::Pagination::Keyset::Iterator + .new(scope: scope, in_operator_optimization_options: { array_scope: array_scope, array_mapping_scope: array_mapping_scope }) + .each_batch(of: BATCH_SIZE) do |batch| + ids = batch.reselect!(:id).to_a.map(&:id) + Ci::JobArtifact.unlocked.where(id: ids).update_all(locked: Ci::JobArtifact.lockeds[:unlocked], expire_at: expiry_time) + end + end + # rubocop:enable CodeReuse/ActiveRecord + + private + + attr_reader :project_id, :expiry_time + end + end +end diff --git a/app/services/ci/pipeline_processing/atomic_processing_service.rb b/app/services/ci/pipeline_processing/atomic_processing_service.rb index d8ce063ffb4..508d9c3f2e1 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service.rb @@ -36,9 +36,7 @@ module Ci update_pipeline! update_statuses_processed! - if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, pipeline.project, default_enabled: :yaml) - Ci::ExpirePipelineCacheService.new.execute(pipeline) - end + Ci::ExpirePipelineCacheService.new.execute(pipeline) true end diff --git a/app/services/ci/pipelines/add_job_service.rb b/app/services/ci/pipelines/add_job_service.rb index 703bb22fb5d..fc852bc3edd 100644 --- a/app/services/ci/pipelines/add_job_service.rb +++ b/app/services/ci/pipelines/add_job_service.rb @@ -39,6 +39,12 @@ module Ci job.pipeline = pipeline job.project = pipeline.project job.ref = pipeline.ref + + # update metadata since it might have been lazily initialised before this call + # metadata is present on `Ci::Processable` + if job.respond_to?(:metadata) && job.metadata + job.metadata.project = pipeline.project + end end end end diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb index e2673c763f3..2d6b6aeee14 100644 --- a/app/services/ci/play_build_service.rb +++ b/app/services/ci/play_build_service.rb @@ -14,7 +14,10 @@ module Ci AfterRequeueJobService.new(project, current_user).execute(build) end else - Ci::Build.retry(build, current_user) + # Retrying in Ci::PlayBuildService is a legacy process that should be removed. + # Instead, callers should explicitly execute Ci::RetryBuildService. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/347493. + build.retryable? ? Ci::Build.retry(build, current_user) : build end end diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 5271c0fe93d..e6ec65fcc91 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -4,14 +4,7 @@ module Ci class ProcessBuildService < BaseService def execute(build, current_status) if valid_statuses_for_build(build).include?(current_status) - if build.schedulable? - build.schedule - elsif build.action? - build.actionize - else - enqueue(build) - end - + process(build) true else build.skip @@ -21,6 +14,16 @@ module Ci private + def process(build) + if build.schedulable? + build.schedule + elsif build.action? + build.actionize + else + enqueue(build) + end + end + def enqueue(build) build.enqueue end diff --git a/app/services/ci/process_sync_events_service.rb b/app/services/ci/process_sync_events_service.rb index 6be8c41dc6a..11ce6e8eeaf 100644 --- a/app/services/ci/process_sync_events_service.rb +++ b/app/services/ci/process_sync_events_service.rb @@ -28,18 +28,16 @@ module Ci return if events.empty? - first = events.first - last_processed = nil + processed_events = [] begin events.each do |event| @sync_class.sync!(event) - last_processed = event + processed_events << event end ensure - # remove events till the one that was last succesfully processed - @sync_event_class.id_in(first.id..last_processed.id).delete_all if last_processed + @sync_event_class.id_in(processed_events).delete_all end end diff --git a/app/services/ci/register_runner_service.rb b/app/services/ci/register_runner_service.rb new file mode 100644 index 00000000000..0a2027e33ce --- /dev/null +++ b/app/services/ci/register_runner_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Ci + class RegisterRunnerService + def execute(registration_token, attributes) + runner_type_attrs = check_token_and_extract_attrs(registration_token) + + return unless runner_type_attrs + + ::Ci::Runner.create(attributes.merge(runner_type_attrs)) + end + + private + + def check_token_and_extract_attrs(registration_token) + if runner_registration_token_valid?(registration_token) + # Create shared runner. Requires admin access + { runner_type: :instance_type } + elsif runner_registrar_valid?('project') && project = ::Project.find_by_runners_token(registration_token) + # Create a specific runner for the project + { runner_type: :project_type, projects: [project] } + elsif runner_registrar_valid?('group') && group = ::Group.find_by_runners_token(registration_token) + # Create a specific runner for the group + { runner_type: :group_type, groups: [group] } + end + end + + def runner_registration_token_valid?(registration_token) + ActiveSupport::SecurityUtils.secure_compare(registration_token, Gitlab::CurrentSettings.runners_registration_token) + end + + def runner_registrar_valid?(type) + Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type) + end + end +end diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index 89fe4ff9f60..7e5d5373648 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -25,10 +25,6 @@ module Ci Gitlab::OptimisticLocking.retry_lock(new_build, name: 'retry_build', &:enqueue) AfterRequeueJobService.new(project, current_user).execute(build) - - ::MergeRequests::AddTodoWhenBuildFailsService - .new(project: project, current_user: current_user) - .close(new_build) end end @@ -42,16 +38,25 @@ module Ci check_access!(build) new_build = clone_build(build) + + new_build.run_after_commit do + ::MergeRequests::AddTodoWhenBuildFailsService + .new(project: project) + .close(new_build) + end + + if create_deployment_in_separate_transaction? + new_build.run_after_commit do |new_build| + ::Deployments::CreateForBuildService.new.execute(new_build) + end + end + ::Ci::Pipelines::AddJobService.new(build.pipeline).execute!(new_build) do |job| BulkInsertableAssociations.with_bulk_insert do job.save! end end - if create_deployment_in_separate_transaction? - clone_deployment!(new_build, build) - end - build.reset # refresh the data to get new values of `retried` and `processed`. new_build @@ -95,20 +100,6 @@ module Ci .deployment_attributes_for(new_build, old_build.persisted_environment) end - def clone_deployment!(new_build, old_build) - return unless old_build.deployment.present? - - # We should clone the previous deployment attributes instead of initializing - # new object with `Seed::Deployment`. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/347206 - deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment - .new(new_build, old_build.persisted_environment).to_resource - - return unless deployment - - new_build.create_deployment!(deployment.attributes) - end - def create_deployment_in_separate_transaction? strong_memoize(:create_deployment_in_separate_transaction) do ::Feature.enabled?(:create_deployment_in_separate_transaction, project, default_enabled: :yaml) diff --git a/app/services/ci/stuck_builds/drop_helpers.rb b/app/services/ci/stuck_builds/drop_helpers.rb index f79b805c23d..048b52c6e13 100644 --- a/app/services/ci/stuck_builds/drop_helpers.rb +++ b/app/services/ci/stuck_builds/drop_helpers.rb @@ -34,7 +34,7 @@ module Ci # rubocop: enable CodeReuse/ActiveRecord def drop_build(type, build, reason) - Gitlab::AppLogger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{build.status}, failure_reason: #{reason})" + log_dropping_message(type, build, reason) Gitlab::OptimisticLocking.retry_lock(build, 3, name: 'stuck_ci_jobs_worker_drop_build') do |b| b.drop(reason) end @@ -53,6 +53,16 @@ module Ci project_id: build.project_id ) end + + def log_dropping_message(type, build, reason) + Gitlab::AppLogger.info(class: self.class.name, + message: "Dropping #{type} build", + build_stuck_type: type, + build_id: build.id, + runner_id: build.runner_id, + build_status: build.status, + build_failure_reason: reason) + end end end end diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb index 146239bb7e5..2e38969c7a9 100644 --- a/app/services/ci/update_build_queue_service.rb +++ b/app/services/ci/update_build_queue_service.rb @@ -99,17 +99,15 @@ module Ci private def tick_for(build, runners) - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do - runners = runners.with_recent_runner_queue - runners = runners.with_tags if Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml) + runners = runners.with_recent_runner_queue + runners = runners.with_tags if Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml) - metrics.observe_active_runners(-> { runners.to_a.size }) + metrics.observe_active_runners(-> { runners.to_a.size }) - runners.each do |runner| - metrics.increment_runner_tick(runner) + runners.each do |runner| + metrics.increment_runner_tick(runner) - runner.pick_build!(build) - end + runner.pick_build!(build) end end diff --git a/app/services/clusters/agent_tokens/create_service.rb b/app/services/clusters/agent_tokens/create_service.rb index 5b8a0e46a6c..2539ffdc5ba 100644 --- a/app/services/clusters/agent_tokens/create_service.rb +++ b/app/services/clusters/agent_tokens/create_service.rb @@ -30,13 +30,14 @@ module Clusters end def log_activity_event!(token) - token.agent.activity_events.create!( + Clusters::Agents::CreateActivityEventService.new( + token.agent, kind: :token_created, level: :info, recorded_at: token.created_at, user: current_user, agent_token: token - ) + ).execute end end end diff --git a/app/services/clusters/agent_tokens/track_usage_service.rb b/app/services/clusters/agent_tokens/track_usage_service.rb new file mode 100644 index 00000000000..fdc79ac0f8b --- /dev/null +++ b/app/services/clusters/agent_tokens/track_usage_service.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Clusters + module AgentTokens + class TrackUsageService + # The `UPDATE_USED_COLUMN_EVERY` defines how often the token DB entry can be updated + UPDATE_USED_COLUMN_EVERY = (40.minutes..55.minutes).freeze + + delegate :agent, to: :token + + def initialize(token) + @token = token + end + + def execute + track_values = { last_used_at: Time.current.utc } + + token.cache_attributes(track_values) + + if can_update_track_values? + log_activity_event!(track_values[:last_used_at]) unless agent.connected? + + # Use update_column so updated_at is skipped + token.update_columns(track_values) + end + end + + private + + attr_reader :token + + def can_update_track_values? + # Use a random threshold to prevent beating DB updates. + last_used_at_max_age = Random.rand(UPDATE_USED_COLUMN_EVERY) + + real_last_used_at = token.read_attribute(:last_used_at) + + # Handle too many updates from high token traffic + real_last_used_at.nil? || + (Time.current - real_last_used_at) >= last_used_at_max_age + end + + def log_activity_event!(recorded_at) + Clusters::Agents::CreateActivityEventService.new( + agent, + kind: :agent_connected, + level: :info, + recorded_at: recorded_at, + agent_token: token + ).execute + end + end + end +end diff --git a/app/services/clusters/agents/create_activity_event_service.rb b/app/services/clusters/agents/create_activity_event_service.rb new file mode 100644 index 00000000000..886dddf1a52 --- /dev/null +++ b/app/services/clusters/agents/create_activity_event_service.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class CreateActivityEventService + def initialize(agent, **params) + @agent = agent + @params = params + end + + def execute + agent.activity_events.create!(params) + + DeleteExpiredEventsWorker.perform_at(schedule_cleanup_at, agent.id) + + ServiceResponse.success + end + + private + + attr_reader :agent, :params + + def schedule_cleanup_at + 1.hour.from_now.change(min: agent.id % 60) + end + end + end +end diff --git a/app/services/clusters/agents/delete_expired_events_service.rb b/app/services/clusters/agents/delete_expired_events_service.rb new file mode 100644 index 00000000000..a0c0291c1fb --- /dev/null +++ b/app/services/clusters/agents/delete_expired_events_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class DeleteExpiredEventsService + def initialize(agent) + @agent = agent + end + + def execute + agent.activity_events + .recorded_before(remove_events_before) + .each_batch { |batch| batch.delete_all } + end + + private + + attr_reader :agent + + def remove_events_before + agent.activity_event_deletion_cutoff + end + end + end +end diff --git a/app/services/concerns/issues/issue_type_helpers.rb b/app/services/concerns/issues/issue_type_helpers.rb index 44c20d20ff1..e6ac08a567d 100644 --- a/app/services/concerns/issues/issue_type_helpers.rb +++ b/app/services/concerns/issues/issue_type_helpers.rb @@ -5,7 +5,7 @@ module Issues # @param object [Issue, Project] # @param issue_type [String, Symbol] def create_issue_type_allowed?(object, issue_type) - WorkItem::Type.base_types.key?(issue_type.to_s) && + WorkItems::Type.base_types.key?(issue_type.to_s) && can?(current_user, :"create_#{issue_type}", object) end end diff --git a/app/services/dependency_proxy/download_blob_service.rb b/app/services/dependency_proxy/download_blob_service.rb deleted file mode 100644 index b3548c8a126..00000000000 --- a/app/services/dependency_proxy/download_blob_service.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module DependencyProxy - class DownloadBlobService < DependencyProxy::BaseService - def initialize(image, blob_sha, token) - @image = image - @blob_sha = blob_sha - @token = token - @temp_file = Tempfile.new - end - - def execute - File.open(@temp_file.path, "wb") do |file| - Gitlab::HTTP.get(blob_url, headers: auth_headers, stream_body: true) do |fragment| - if [301, 302, 307].include?(fragment.code) - # do nothing - elsif fragment.code == 200 - file.write(fragment) - else - raise DownloadError.new('Non-success response code on downloading blob fragment', fragment.code) - end - end - end - - success(file: @temp_file) - rescue DownloadError => exception - error(exception.message, exception.http_status) - rescue Timeout::Error => exception - error(exception.message, 599) - end - - private - - def blob_url - registry.blob_url(@image, @blob_sha) - end - end -end diff --git a/app/services/dependency_proxy/find_or_create_blob_service.rb b/app/services/dependency_proxy/find_or_create_blob_service.rb deleted file mode 100644 index 1b43263a3ba..00000000000 --- a/app/services/dependency_proxy/find_or_create_blob_service.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module DependencyProxy - class FindOrCreateBlobService < DependencyProxy::BaseService - def initialize(group, image, token, blob_sha) - @group = group - @image = image - @token = token - @blob_sha = blob_sha - end - - def execute - from_cache = true - file_name = @blob_sha.sub('sha256:', '') + '.gz' - blob = @group.dependency_proxy_blobs.active.find_or_build(file_name) - - unless blob.persisted? - from_cache = false - result = DependencyProxy::DownloadBlobService - .new(@image, @blob_sha, @token).execute - - if result[:status] == :error - log_failure(result) - - return error('Failed to download the blob', result[:http_status]) - end - - blob.file = result[:file] - blob.size = result[:file].size - blob.save! - end - - blob.read! if from_cache - success(blob: blob, from_cache: from_cache) - end - - private - - def log_failure(result) - log_error( - "Dependency proxy: Failed to download the blob." \ - "Blob sha: #{@blob_sha}." \ - "Error message: #{result[:message][0, 100]}" \ - "HTTP status: #{result[:http_status]}" - ) - end - end -end diff --git a/app/services/deployments/archive_in_project_service.rb b/app/services/deployments/archive_in_project_service.rb index a593721f390..d80ed637cd8 100644 --- a/app/services/deployments/archive_in_project_service.rb +++ b/app/services/deployments/archive_in_project_service.rb @@ -7,10 +7,6 @@ module Deployments BATCH_SIZE = 100 def execute - unless ::Feature.enabled?(:deployments_archive, project, default_enabled: :yaml) - return error('Feature flag is not enabled') - end - deployments = Deployment.archivables_in(project, limit: BATCH_SIZE) return success(result: :empty) if deployments.empty? diff --git a/app/services/deployments/create_for_build_service.rb b/app/services/deployments/create_for_build_service.rb new file mode 100644 index 00000000000..b3e2d2edb59 --- /dev/null +++ b/app/services/deployments/create_for_build_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Deployments + # This class creates a deployment record for a build (a pipeline job). + class CreateForBuildService + DeploymentCreationError = Class.new(StandardError) + + def execute(build) + return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? + + # TODO: Move all buisness logic in `Seed::Deployment` to this class after + # `create_deployment_in_separate_transaction` feature flag has been removed. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/348778 + deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment + .new(build, build.persisted_environment).to_resource + + return unless deployment + + build.create_deployment!(deployment.attributes) + rescue ActiveRecord::RecordInvalid => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + DeploymentCreationError.new(e.message), build_id: build.id) + end + end +end diff --git a/app/services/deployments/update_environment_service.rb b/app/services/deployments/update_environment_service.rb index 83c37257297..19b950044d0 100644 --- a/app/services/deployments/update_environment_service.rb +++ b/app/services/deployments/update_environment_service.rb @@ -26,7 +26,7 @@ module Deployments end def update_environment(deployment) - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases + ApplicationRecord.transaction do # Renew attributes at update renew_external_url renew_auto_stop_in diff --git a/app/services/design_management/copy_design_collection/copy_service.rb b/app/services/design_management/copy_design_collection/copy_service.rb index 5e557e9ea53..886077191ab 100644 --- a/app/services/design_management/copy_design_collection/copy_service.rb +++ b/app/services/design_management/copy_design_collection/copy_service.rb @@ -16,7 +16,7 @@ module DesignManagement @temporary_branch = "CopyDesignCollectionService_#{SecureRandom.hex}" # The user who triggered the copy may not have permissions to push # to the design repository. - @git_user = @target_project.default_owner + @git_user = @target_project.first_owner @designs = DesignManagement::Design.unscoped.where(issue: issue).order(:id).load @versions = DesignManagement::Version.unscoped.where(issue: issue).order(:id).includes(:designs).load diff --git a/app/services/emails/confirm_service.rb b/app/services/emails/confirm_service.rb index 38204e011dd..4ceff4e3cfa 100644 --- a/app/services/emails/confirm_service.rb +++ b/app/services/emails/confirm_service.rb @@ -7,3 +7,5 @@ module Emails end end end + +Emails::ConfirmService.prepend_mod diff --git a/app/services/error_tracking/collect_error_service.rb b/app/services/error_tracking/collect_error_service.rb index 304e3898ee5..50508c9810a 100644 --- a/app/services/error_tracking/collect_error_service.rb +++ b/app/services/error_tracking/collect_error_service.rb @@ -2,6 +2,8 @@ module ErrorTracking class CollectErrorService < ::BaseService + include Gitlab::Utils::StrongMemoize + def execute # Error is a way to group events based on common data like name or cause # of exception. We need to keep a sane balance here between taking too little @@ -43,16 +45,29 @@ module ErrorTracking end def exception - event['exception']['values'].first + strong_memoize(:exception) do + # Find the first exception that has a stacktrace since the first + # exception may not provide adequate context (e.g. in the Go SDK). + entries = event['exception']['values'] + entries.find { |x| x.key?('stacktrace') } || entries.first + end + end + + def stacktrace_frames + strong_memoize(:stacktrace_frames) do + exception.dig('stacktrace', 'frames') + end end def actor return event['transaction'] if event['transaction'] - # Some SDK do not have transaction attribute. + # Some SDKs do not have a transaction attribute. # So we build it by combining function name and module name from # the last item in stacktrace. - last_line = exception.dig('stacktrace', 'frames').last + return unless stacktrace_frames.present? + + last_line = stacktrace_frames.last "#{last_line['function']}(#{last_line['module']})" end diff --git a/app/services/events/destroy_service.rb b/app/services/events/destroy_service.rb index fdb718f0fcb..3a0a7b339bf 100644 --- a/app/services/events/destroy_service.rb +++ b/app/services/events/destroy_service.rb @@ -2,20 +2,29 @@ module Events class DestroyService + BATCH_SIZE = 50 + def initialize(project) @project = project end def execute - project.events.all.delete_all + loop do + count = delete_events_in_batches + break if count < BATCH_SIZE + end ServiceResponse.success(message: 'Events were deleted.') - rescue StandardError - ServiceResponse.error(message: 'Failed to remove events.') + rescue StandardError => e + ServiceResponse.error(message: e.message) end private attr_reader :project + + def delete_events_in_batches + project.events.limit(BATCH_SIZE).delete_all + end end end diff --git a/app/services/google_cloud/create_service_accounts_service.rb b/app/services/google_cloud/create_service_accounts_service.rb new file mode 100644 index 00000000000..fa025e8f672 --- /dev/null +++ b/app/services/google_cloud/create_service_accounts_service.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module GoogleCloud + class CreateServiceAccountsService < :: BaseService + def execute + service_account = google_api_client.create_service_account(gcp_project_id, service_account_name, service_account_desc) + service_account_key = google_api_client.create_service_account_key(gcp_project_id, service_account.unique_id) + + service_accounts_service.add_for_project( + environment_name, + service_account.project_id, + service_account.to_json, + service_account_key.to_json, + environment_protected? + ) + + ServiceResponse.success(message: _('Service account generated successfully'), payload: { + service_account: service_account, + service_account_key: service_account_key + }) + end + + private + + def google_oauth2_token + @params[:google_oauth2_token] + end + + def gcp_project_id + @params[:gcp_project_id] + end + + def environment_name + @params[:environment_name] + end + + def google_api_client + GoogleApi::CloudPlatform::Client.new(google_oauth2_token, nil) + end + + def service_accounts_service + GoogleCloud::ServiceAccountsService.new(project) + end + + def service_account_name + "GitLab :: #{project.name} :: #{environment_name}" + end + + def service_account_desc + "GitLab generated service account for project '#{project.name}' and environment '#{environment_name}'" + end + + # Overriden in EE + def environment_protected? + false + end + end +end + +GoogleCloud::CreateServiceAccountsService.prepend_mod diff --git a/app/services/google_cloud/service_accounts_service.rb b/app/services/google_cloud/service_accounts_service.rb index a512b27493d..3014daf08e2 100644 --- a/app/services/google_cloud/service_accounts_service.rb +++ b/app/services/google_cloud/service_accounts_service.rb @@ -27,39 +27,42 @@ module GoogleCloud end end - def add_for_project(environment, gcp_project_id, service_account, service_account_key) + def add_for_project(environment, gcp_project_id, service_account, service_account_key, is_protected) project_var_create_or_replace( environment, 'GCP_PROJECT_ID', - gcp_project_id + gcp_project_id, + is_protected ) project_var_create_or_replace( environment, 'GCP_SERVICE_ACCOUNT', - service_account + service_account, + is_protected ) project_var_create_or_replace( environment, 'GCP_SERVICE_ACCOUNT_KEY', - service_account_key + service_account_key, + is_protected ) end private def group_vars_by_environment - filtered_vars = @project.variables.filter { |variable| GCP_KEYS.include? variable.key } + filtered_vars = project.variables.filter { |variable| GCP_KEYS.include? variable.key } filtered_vars.each_with_object({}) do |variable, grouped| grouped[variable.environment_scope] ||= {} grouped[variable.environment_scope][variable.key] = variable.value end end - def project_var_create_or_replace(environment_scope, key, value) + def project_var_create_or_replace(environment_scope, key, value, is_protected) params = { key: key, filter: { environment_scope: environment_scope } } - existing_variable = ::Ci::VariablesFinder.new(@project, params).execute.first + existing_variable = ::Ci::VariablesFinder.new(project, params).execute.first existing_variable.destroy if existing_variable - @project.variables.create!(key: key, value: value, environment_scope: environment_scope, protected: true) + project.variables.create!(key: key, value: value, environment_scope: environment_scope, protected: is_protected) end end end diff --git a/app/services/groups/update_service.rb b/app/services/groups/update_service.rb index 2d6334251ad..b3b0397eac3 100644 --- a/app/services/groups/update_service.rb +++ b/app/services/groups/update_service.rb @@ -107,6 +107,7 @@ module Groups def handle_changes handle_settings_update + handle_crm_settings_update unless params[:crm_enabled].nil? end def handle_settings_update @@ -116,6 +117,15 @@ module Groups ::NamespaceSettings::UpdateService.new(current_user, group, settings_params).execute end + def handle_crm_settings_update + crm_enabled = params.delete(:crm_enabled) + return if group.crm_enabled? == crm_enabled + + crm_settings = group.crm_settings || group.build_crm_settings + crm_settings.enabled = crm_enabled + crm_settings.save + end + def allowed_settings_params SETTINGS_PARAMS end diff --git a/app/services/import/gitlab_projects/create_project_from_remote_file_service.rb b/app/services/import/gitlab_projects/create_project_from_remote_file_service.rb index bbfdaf692f9..edb9dc8ad91 100644 --- a/app/services/import/gitlab_projects/create_project_from_remote_file_service.rb +++ b/app/services/import/gitlab_projects/create_project_from_remote_file_service.rb @@ -4,7 +4,10 @@ module Import module GitlabProjects class CreateProjectFromRemoteFileService < CreateProjectFromUploadedFileService FILE_SIZE_LIMIT = 10.gigabytes - ALLOWED_CONTENT_TYPES = ['application/gzip'].freeze + ALLOWED_CONTENT_TYPES = [ + 'application/gzip', # most common content-type when fetching a tar.gz + 'application/x-tar' # aws-s3 uses x-tar for tar.gz files + ].freeze validate :valid_remote_import_url? validate :validate_file_size @@ -44,17 +47,27 @@ module Import end def validate_content_type + # AWS-S3 presigned URLs don't respond to HTTP HEAD requests, + # so file type cannot be validated + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75170#note_748059103 + return if amazon_s3? + if headers['content-type'].blank? errors.add(:base, "Missing 'ContentType' header") elsif !ALLOWED_CONTENT_TYPES.include?(headers['content-type']) errors.add(:base, "Remote file content type '%{content_type}' not allowed. (Allowed content types: %{allowed})" % { content_type: headers['content-type'], - allowed: ALLOWED_CONTENT_TYPES.join(',') + allowed: ALLOWED_CONTENT_TYPES.join(', ') }) end end def validate_file_size + # AWS-S3 presigned URLs don't respond to HTTP HEAD requests, + # so file size cannot be validated + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75170#note_748059103 + return if amazon_s3? + if headers['content-length'].to_i == 0 errors.add(:base, "Missing 'ContentLength' header") elsif headers['content-length'].to_i > FILE_SIZE_LIMIT @@ -64,6 +77,10 @@ module Import end end + def amazon_s3? + headers['Server'] == 'AmazonS3' && headers['x-amz-request-id'].present? + end + def headers return {} if params[:remote_import_url].blank? || !valid_remote_import_url? diff --git a/app/services/import/validate_remote_git_endpoint_service.rb b/app/services/import/validate_remote_git_endpoint_service.rb index afccb5373a9..1b8fa45e979 100644 --- a/app/services/import/validate_remote_git_endpoint_service.rb +++ b/app/services/import/validate_remote_git_endpoint_service.rb @@ -23,6 +23,8 @@ module Import return ServiceResponse.error(message: "#{@params[:url]} is not a valid URL") unless uri + return ServiceResponse.success if uri.scheme == 'git' + uri.fragment = nil url = Gitlab::Utils.append_path(uri.to_s, "/info/refs?service=#{GIT_SERVICE_NAME}") diff --git a/app/services/incident_management/issuable_escalation_statuses/after_update_service.rb b/app/services/incident_management/issuable_escalation_statuses/after_update_service.rb new file mode 100644 index 00000000000..a7a99f88b32 --- /dev/null +++ b/app/services/incident_management/issuable_escalation_statuses/after_update_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module IncidentManagement + module IssuableEscalationStatuses + class AfterUpdateService < ::BaseProjectService + def initialize(issuable, current_user) + @issuable = issuable + @escalation_status = issuable.escalation_status + @alert = issuable.alert_management_alert + + super(project: issuable.project, current_user: current_user) + end + + def execute + after_update + + ServiceResponse.success(payload: { escalation_status: escalation_status }) + end + + private + + attr_reader :issuable, :escalation_status, :alert + + def after_update + sync_to_alert + end + + def sync_to_alert + return unless alert + return if alert.status == escalation_status.status + + ::AlertManagement::Alerts::UpdateService.new( + alert, + current_user, + status: escalation_status.status_name + ).execute + end + end + end +end + +::IncidentManagement::IssuableEscalationStatuses::AfterUpdateService.prepend_mod diff --git a/app/services/incident_management/issuable_escalation_statuses/prepare_update_service.rb b/app/services/incident_management/issuable_escalation_statuses/prepare_update_service.rb new file mode 100644 index 00000000000..1a660e1a163 --- /dev/null +++ b/app/services/incident_management/issuable_escalation_statuses/prepare_update_service.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module IncidentManagement + module IssuableEscalationStatuses + class PrepareUpdateService + include Gitlab::Utils::StrongMemoize + + SUPPORTED_PARAMS = %i[status].freeze + + InvalidParamError = Class.new(StandardError) + + def initialize(issuable, current_user, params) + @issuable = issuable + @current_user = current_user + @params = params.dup || {} + @project = issuable.project + end + + def execute + return availability_error unless available? + + filter_unsupported_params + filter_attributes + filter_redundant_params + + ServiceResponse.success(payload: { escalation_status: params }) + rescue InvalidParamError + invalid_param_error + end + + private + + attr_reader :issuable, :current_user, :params, :project + + def available? + Feature.enabled?(:incident_escalations, project) && + user_has_permissions? && + issuable.supports_escalation? && + escalation_status.present? + end + + def user_has_permissions? + current_user&.can?(:update_escalation_status, issuable) + end + + def escalation_status + strong_memoize(:escalation_status) do + issuable.escalation_status + end + end + + def filter_unsupported_params + params.slice!(*supported_params) + end + + def supported_params + SUPPORTED_PARAMS + end + + def filter_attributes + filter_status + end + + def filter_status + status = params.delete(:status) + return unless status + + status_event = escalation_status.status_event_for(status) + raise InvalidParamError unless status_event + + params[:status_event] = status_event + end + + def filter_redundant_params + params.delete_if do |key, value| + current_params.key?(key) && current_params[key] == value + end + end + + def current_params + strong_memoize(:current_params) do + { + status_event: escalation_status.status_event_for(escalation_status.status_name) + } + end + end + + def availability_error + ServiceResponse.error(message: 'Escalation status updates are not available for this issue, user, or project.') + end + + def invalid_param_error + ServiceResponse.error(message: 'Invalid value was provided for a parameter.') + end + end + end +end + +::IncidentManagement::IssuableEscalationStatuses::PrepareUpdateService.prepend_mod diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 1d1d9b6bec7..ecf10cf97a8 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -63,6 +63,7 @@ class IssuableBaseService < ::BaseProjectService filter_milestone filter_labels filter_severity(issuable) + filter_escalation_status(issuable) end def filter_assignees(issuable) @@ -152,6 +153,18 @@ class IssuableBaseService < ::BaseProjectService params[:issuable_severity_attributes] = { severity: severity } end + def filter_escalation_status(issuable) + result = ::IncidentManagement::IssuableEscalationStatuses::PrepareUpdateService.new( + issuable, + current_user, + params.delete(:escalation_status) + ).execute + + return unless result.success? && result.payload.present? + + params[:incident_management_issuable_escalation_status_attributes] = result[:escalation_status] + end + def process_label_ids(attributes, existing_label_ids: nil, extra_label_ids: []) label_ids = attributes.delete(:label_ids) add_label_ids = attributes.delete(:add_label_ids) @@ -471,6 +484,7 @@ class IssuableBaseService < ::BaseProjectService associations[:description] = issuable.description associations[:reviewers] = issuable.reviewers.to_a if issuable.allows_reviewers? associations[:severity] = issuable.severity if issuable.supports_severity? + associations[:escalation_status] = issuable.escalation_status&.slice(:status, :policy_id) if issuable.supports_escalation? associations end diff --git a/app/services/issues/base_service.rb b/app/services/issues/base_service.rb index 577f7dd1e3a..37d667d4be8 100644 --- a/app/services/issues/base_service.rb +++ b/app/services/issues/base_service.rb @@ -36,8 +36,8 @@ module Issues private def find_work_item_type_id(issue_type) - work_item_type = WorkItem::Type.default_by_type(issue_type) - work_item_type ||= WorkItem::Type.default_issue_type + work_item_type = WorkItems::Type.default_by_type(issue_type) + work_item_type ||= WorkItems::Type.default_issue_type work_item_type.id end @@ -46,7 +46,6 @@ module Issues super params.delete(:issue_type) unless create_issue_type_allowed?(issue, params[:issue_type]) - filter_incident_label(issue) if params[:issue_type] moved_issue = params.delete(:moved_issue) @@ -89,37 +88,6 @@ module Issues Milestones::IssuesCountService.new(milestone).delete_cache end - - # @param issue [Issue] - def filter_incident_label(issue) - return unless add_incident_label?(issue) || remove_incident_label?(issue) - - label = ::IncidentManagement::CreateIncidentLabelService - .new(project, current_user) - .execute - .payload[:label] - - # These(add_label_ids, remove_label_ids) are being added ahead of time - # to be consumed by #process_label_ids, this allows system notes - # to be applied correctly alongside the label updates. - if add_incident_label?(issue) - params[:add_label_ids] ||= [] - params[:add_label_ids] << label.id - else - params[:remove_label_ids] ||= [] - params[:remove_label_ids] << label.id - end - end - - # @param issue [Issue] - def add_incident_label?(issue) - issue.incident? - end - - # @param _issue [Issue, nil] - def remove_incident_label?(_issue) - false - end end end diff --git a/app/services/issues/build_service.rb b/app/services/issues/build_service.rb index 8fd844c4886..1ebf9bb6ba2 100644 --- a/app/services/issues/build_service.rb +++ b/app/services/issues/build_service.rb @@ -7,7 +7,7 @@ module Issues def execute filter_resolve_discussion_params - @issue = project.issues.new(issue_params).tap do |issue| + @issue = model_klass.new(issue_params.merge(project: project)).tap do |issue| ensure_milestone_available(issue) end end @@ -62,16 +62,25 @@ module Issues def issue_params @issue_params ||= build_issue_params - # If :issue_type is nil then params[:issue_type] was either nil - # or not permitted. Either way, the :issue_type will default - # to the column default of `issue`. And that means we need to - # ensure the work_item_type_id is set - @issue_params[:work_item_type_id] = get_work_item_type_id(@issue_params[:issue_type]) + if @issue_params[:work_item_type].present? + @issue_params[:issue_type] = @issue_params[:work_item_type].base_type + else + # If :issue_type is nil then params[:issue_type] was either nil + # or not permitted. Either way, the :issue_type will default + # to the column default of `issue`. And that means we need to + # ensure the work_item_type_id is set + @issue_params[:work_item_type_id] = get_work_item_type_id(@issue_params[:issue_type]) + end + @issue_params end private + def model_klass + ::Issue + end + def allowed_issue_params allowed_params = [ :title, @@ -79,8 +88,11 @@ module Issues :confidential ] + params[:work_item_type] = WorkItems::Type.find_by(id: params[:work_item_type_id]) if params[:work_item_type_id].present? # rubocop: disable CodeReuse/ActiveRecord + allowed_params << :milestone_id if can?(current_user, :admin_issue, project) allowed_params << :issue_type if create_issue_type_allowed?(project, params[:issue_type]) + allowed_params << :work_item_type if create_issue_type_allowed?(project, params[:work_item_type]&.base_type) params.slice(*allowed_params) end diff --git a/app/services/issues/create_service.rb b/app/services/issues/create_service.rb index 79b59eee5e1..e29bcf4a453 100644 --- a/app/services/issues/create_service.rb +++ b/app/services/issues/create_service.rb @@ -12,13 +12,14 @@ module Issues # spam_checking is likely to be necessary. However, if there is not a request available in scope # in the caller (for example, an issue created via email) and the required arguments to the # SpamParams constructor are not otherwise available, spam_params: must be explicitly passed as nil. - def initialize(project:, current_user: nil, params: {}, spam_params:) + def initialize(project:, current_user: nil, params: {}, spam_params:, build_service: nil) super(project: project, current_user: current_user, params: params) @spam_params = spam_params + @build_service = build_service || BuildService.new(project: project, current_user: current_user, params: params) end def execute(skip_system_notes: false) - @issue = BuildService.new(project: project, current_user: current_user, params: params).execute + @issue = @build_service.execute filter_resolve_discussion_params diff --git a/app/services/issues/set_crm_contacts_service.rb b/app/services/issues/set_crm_contacts_service.rb index c435ab81b4d..947d46f0809 100644 --- a/app/services/issues/set_crm_contacts_service.rb +++ b/app/services/issues/set_crm_contacts_service.rb @@ -48,7 +48,7 @@ module Issues end def add_by_email - contact_ids = ::CustomerRelations::Contact.find_ids_by_emails(project_group.id, params[:add_emails]) + contact_ids = ::CustomerRelations::Contact.find_ids_by_emails(project_group, params[:add_emails]) add_by_id(contact_ids) end diff --git a/app/services/issues/update_service.rb b/app/services/issues/update_service.rb index 824a609dfb9..aecb22453b7 100644 --- a/app/services/issues/update_service.rb +++ b/app/services/issues/update_service.rb @@ -53,6 +53,7 @@ module Issues old_mentioned_users = old_associations.fetch(:mentioned_users, []) old_assignees = old_associations.fetch(:assignees, []) old_severity = old_associations[:severity] + old_escalation_status = old_associations[:escalation_status] if has_changes?(issue, old_labels: old_labels, old_assignees: old_assignees) todo_service.resolve_todos_for_target(issue, current_user) @@ -69,6 +70,7 @@ module Issues handle_milestone_change(issue) handle_added_mentions(issue, old_mentioned_users) handle_severity_change(issue, old_severity) + handle_escalation_status_change(issue, old_escalation_status) handle_issue_type_change(issue) end @@ -208,6 +210,13 @@ module Issues ::IncidentManagement::AddSeveritySystemNoteWorker.perform_async(issue.id, current_user.id) end + def handle_escalation_status_change(issue, old_escalation_status) + return unless old_escalation_status.present? + return if issue.escalation_status&.slice(:status, :policy_id) == old_escalation_status + + ::IncidentManagement::IssuableEscalationStatuses::AfterUpdateService.new(issue, current_user).execute + end + # rubocop: disable CodeReuse/ActiveRecord def issuable_for_positioning(id, board_group_id = nil) return unless id @@ -227,16 +236,6 @@ module Issues SystemNoteService.change_issue_confidentiality(issue, issue.project, current_user) end - override :add_incident_label? - def add_incident_label?(issue) - issue.issue_type != params[:issue_type] && !issue.incident? - end - - override :remove_incident_label? - def remove_incident_label?(issue) - issue.issue_type != params[:issue_type] && issue.incident? - end - def handle_issue_type_change(issue) return unless issue.previous_changes.include?('issue_type') @@ -245,6 +244,8 @@ module Issues def do_handle_issue_type_change(issue) SystemNoteService.change_issue_type(issue, current_user) + + ::IncidentManagement::IssuableEscalationStatuses::CreateService.new(issue).execute if issue.supports_escalation? end end end diff --git a/app/services/labels/transfer_service.rb b/app/services/labels/transfer_service.rb index 19d419609a5..67163cb8122 100644 --- a/app/services/labels/transfer_service.rb +++ b/app/services/labels/transfer_service.rb @@ -50,32 +50,17 @@ module Labels # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_issues - @labels_applied_to_issues ||= if use_optimized_group_labels_query? - Label.joins(:issues) - .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) - .where(issues: { project_id: project.id }).reorder(nil) - else - Label.joins(:issues).where( - issues: { project_id: project.id }, - labels: { group_id: old_group.self_and_ancestors } - ) - end + @labels_applied_to_issues ||= Label.joins(:issues) + .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) + .where(issues: { project_id: project.id }).reorder(nil) end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_merge_requests - @labels_applied_to_mrs ||= if use_optimized_group_labels_query? - Label.joins(:merge_requests) - .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) - .where(merge_requests: { target_project_id: project.id }).reorder(nil) - else - Label.joins(:merge_requests) - .where( - merge_requests: { target_project_id: project.id }, - labels: { group_id: old_group.self_and_ancestors } - ) - end + @labels_applied_to_mrs ||= Label.joins(:merge_requests) + .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) + .where(merge_requests: { target_project_id: project.id }).reorder(nil) end # rubocop: enable CodeReuse/ActiveRecord @@ -99,9 +84,5 @@ module Labels .update_all(label_id: new_label_id) end # rubocop: enable CodeReuse/ActiveRecord - - def use_optimized_group_labels_query? - Feature.enabled?(:use_optimized_group_labels_query, project.root_namespace, default_enabled: :yaml) - end end end diff --git a/app/services/loose_foreign_keys/process_deleted_records_service.rb b/app/services/loose_foreign_keys/process_deleted_records_service.rb index 025829aa774..2826bdb4c3c 100644 --- a/app/services/loose_foreign_keys/process_deleted_records_service.rb +++ b/app/services/loose_foreign_keys/process_deleted_records_service.rb @@ -10,7 +10,6 @@ module LooseForeignKeys def execute modification_tracker = ModificationTracker.new - tracked_tables.cycle do |table| records = load_batch_for_table(table) diff --git a/app/services/merge_requests/add_todo_when_build_fails_service.rb b/app/services/merge_requests/add_todo_when_build_fails_service.rb index d3ef892875b..47cd19e9d8d 100644 --- a/app/services/merge_requests/add_todo_when_build_fails_service.rb +++ b/app/services/merge_requests/add_todo_when_build_fails_service.rb @@ -16,9 +16,7 @@ module MergeRequests # build is retried # def close(commit_status) - pipeline_merge_requests(commit_status.pipeline) do |merge_request| - todo_service.merge_request_build_retried(merge_request) - end + close_all(commit_status.pipeline) end def close_all(pipeline) diff --git a/app/services/merge_requests/base_service.rb b/app/services/merge_requests/base_service.rb index d744881549a..b0d0c32abd1 100644 --- a/app/services/merge_requests/base_service.rb +++ b/app/services/merge_requests/base_service.rb @@ -59,7 +59,9 @@ module MergeRequests merge_request_activity_counter.track_users_review_requested(users: new_reviewers) merge_request_activity_counter.track_reviewers_changed_action(user: current_user) - remove_attention_requested(merge_request, current_user) + unless new_reviewers.include?(current_user) + remove_attention_requested(merge_request, current_user) + end end def cleanup_environments(merge_request) diff --git a/app/services/merge_requests/handle_assignees_change_service.rb b/app/services/merge_requests/handle_assignees_change_service.rb index 1d9f7ab59f4..97be9fe8d9f 100644 --- a/app/services/merge_requests/handle_assignees_change_service.rb +++ b/app/services/merge_requests/handle_assignees_change_service.rb @@ -23,7 +23,9 @@ module MergeRequests execute_assignees_hooks(merge_request, old_assignees) if options[:execute_hooks] - remove_attention_requested(merge_request, current_user) + unless new_assignees.include?(current_user) + remove_attention_requested(merge_request, current_user) + end end private diff --git a/app/services/merge_requests/merge_base_service.rb b/app/services/merge_requests/merge_base_service.rb index 3b9d3bccacf..3e630d40b3d 100644 --- a/app/services/merge_requests/merge_base_service.rb +++ b/app/services/merge_requests/merge_base_service.rb @@ -56,7 +56,7 @@ module MergeRequests def commit_message params[:commit_message] || - merge_request.default_merge_commit_message + merge_request.default_merge_commit_message(user: current_user) end def squash_sha! diff --git a/app/services/merge_requests/remove_approval_service.rb b/app/services/merge_requests/remove_approval_service.rb index 872e7e0c89c..198a21884b8 100644 --- a/app/services/merge_requests/remove_approval_service.rb +++ b/app/services/merge_requests/remove_approval_service.rb @@ -17,6 +17,7 @@ module MergeRequests reset_approvals_cache(merge_request) create_note(merge_request) merge_request_activity_counter.track_unapprove_mr_action(user: current_user) + remove_attention_requested(merge_request, current_user) end success diff --git a/app/services/merge_requests/squash_service.rb b/app/services/merge_requests/squash_service.rb index 90cf4af7e41..69b9740c2a5 100644 --- a/app/services/merge_requests/squash_service.rb +++ b/app/services/merge_requests/squash_service.rb @@ -39,7 +39,7 @@ module MergeRequests end def message - params[:squash_commit_message].presence || merge_request.default_squash_commit_message + params[:squash_commit_message].presence || merge_request.default_squash_commit_message(user: current_user) end end end diff --git a/app/services/packages/maven/metadata/sync_service.rb b/app/services/packages/maven/metadata/sync_service.rb index 48e157d4930..4f35db36fb0 100644 --- a/app/services/packages/maven/metadata/sync_service.rb +++ b/app/services/packages/maven/metadata/sync_service.rb @@ -93,10 +93,15 @@ module Packages def metadata_package_file_for(package) return unless package - package.package_files - .with_file_name(Metadata.filename) - .recent - .first + package_files = if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + package.installable_package_files + else + package.package_files + end + + package_files.with_file_name(Metadata.filename) + .recent + .first end def versionless_package_named(name) diff --git a/app/services/packages/terraform_module/create_package_service.rb b/app/services/packages/terraform_module/create_package_service.rb index 03f749edfa8..d1bc79089a3 100644 --- a/app/services/packages/terraform_module/create_package_service.rb +++ b/app/services/packages/terraform_module/create_package_service.rb @@ -7,7 +7,7 @@ module Packages def execute return error('Version is empty.', 400) if params[:module_version].blank? - return error('Package already exists.', 403) if current_package_exists_elsewhere? + return error('Access Denied', 403) if current_package_exists_elsewhere? return error('Package version already exists.', 403) if current_package_version_exists? return error('File is too large.', 400) if file_size_exceeded? diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index aef92b8adee..5c4a0e947de 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -41,6 +41,8 @@ module Projects true rescue StandardError => error + context = Gitlab::ApplicationContext.current.merge(project_id: project.id) + Gitlab::ErrorTracking.track_exception(error, **context) attempt_rollback(project, error.message) false rescue Exception => error # rubocop:disable Lint/RescueException @@ -80,8 +82,14 @@ module Projects end def remove_events + log_info("Attempting to destroy events from #{project.full_path} (#{project.id})") + response = ::Events::DestroyService.new(project).execute + if response.error? + log_error("Event deletion failed on #{project.full_path} with the following message: #{response.message}") + end + response.success? end diff --git a/app/services/projects/fork_service.rb b/app/services/projects/fork_service.rb index 74b7d18f401..3e8d6563709 100644 --- a/app/services/projects/fork_service.rb +++ b/app/services/projects/fork_service.rb @@ -69,6 +69,8 @@ module Projects new_params[:avatar] = @project.avatar end + new_params[:mr_default_target_self] = target_mr_default_target_self unless target_mr_default_target_self.nil? + new_params.merge!(@project.object_pool_params) new_params @@ -127,5 +129,9 @@ module Projects Gitlab::VisibilityLevel.closest_allowed_level(target_level) end + + def target_mr_default_target_self + @target_mr_default_target_self ||= params[:mr_default_target_self] + end end end diff --git a/app/services/projects/overwrite_project_service.rb b/app/services/projects/overwrite_project_service.rb index 2612001eb95..c58fba33b2a 100644 --- a/app/services/projects/overwrite_project_service.rb +++ b/app/services/projects/overwrite_project_service.rb @@ -11,7 +11,9 @@ module Projects move_before_destroy_relationships(source_project) # Reset is required in order to get the proper # uncached fork network method calls value. - destroy_old_project(source_project.reset) + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/340256') do + destroy_old_project(source_project.reset) + end rename_project(source_project.name, source_project.path) @project diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index 56f65718d24..bc517ee3d6f 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -18,6 +18,14 @@ module Projects SUPPORTED_VERSION = '4' + # If feature flag :prometheus_notify_max_alerts is enabled truncate + # alerts to 100 and process only them. + # If feature flag is disabled process any amount of alerts. + # + # This is to mitigate incident: + # https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6086 + PROCESS_MAX_ALERTS = 100 + def initialize(project, payload) @project = project @payload = payload @@ -28,6 +36,8 @@ module Projects return unprocessable_entity unless self.class.processable?(payload) return unauthorized unless valid_alert_manager_token?(token, integration) + truncate_alerts! if max_alerts_exceeded? + alert_responses = process_prometheus_alerts alert_response(alert_responses) @@ -49,12 +59,23 @@ module Projects Gitlab::Utils::DeepSize.new(payload).valid? end - def firings - @firings ||= alerts_by_status('firing') + def max_alerts_exceeded? + return false unless Feature.enabled?(:prometheus_notify_max_alerts, project, type: :ops) + + alerts.size > PROCESS_MAX_ALERTS end - def alerts_by_status(status) - alerts.select { |alert| alert['status'] == status } + def truncate_alerts! + Gitlab::AppLogger.warn( + message: 'Prometheus payload exceeded maximum amount of alerts. Truncating alerts.', + project_id: project.id, + alerts: { + total: alerts.size, + max: PROCESS_MAX_ALERTS + } + ) + + payload['alerts'] = alerts.first(PROCESS_MAX_ALERTS) end def alerts @@ -137,7 +158,7 @@ module Projects end def alert_response(alert_responses) - alerts = alert_responses.map { |resp| resp.payload[:alert] }.compact + alerts = alert_responses.flat_map { |resp| resp.payload[:alerts] }.compact success(alerts) end diff --git a/app/services/projects/update_pages_configuration_service.rb b/app/services/projects/update_pages_configuration_service.rb deleted file mode 100644 index 4272e1dc8b6..00000000000 --- a/app/services/projects/update_pages_configuration_service.rb +++ /dev/null @@ -1,109 +0,0 @@ -# frozen_string_literal: true - -module Projects - class UpdatePagesConfigurationService < BaseService - include Gitlab::Utils::StrongMemoize - - attr_reader :project - - def initialize(project) - @project = project - end - - def execute - return success unless ::Settings.pages.local_store.enabled - - # If the pages were never deployed, we can't write out the config, as the - # directory would not exist. - # https://gitlab.com/gitlab-org/gitlab/-/issues/235139 - return success unless project.pages_deployed? - - unless file_equals?(pages_config_file, pages_config_json) - update_file(pages_config_file, pages_config_json) - reload_daemon - end - - success - end - - private - - def pages_config_json - strong_memoize(:pages_config_json) do - pages_config.to_json - end - end - - def pages_config - { - domains: pages_domains_config, - https_only: project.pages_https_only?, - id: project.project_id, - access_control: !project.public_pages? - } - end - - def pages_domains_config - enabled_pages_domains.map do |domain| - { - domain: domain.domain, - certificate: domain.certificate, - key: domain.key, - https_only: project.pages_https_only? && domain.https?, - id: project.project_id, - access_control: !project.public_pages? - } - end - end - - def enabled_pages_domains - if Gitlab::CurrentSettings.pages_domain_verification_enabled? - project.pages_domains.enabled - else - project.pages_domains - end - end - - def reload_daemon - # GitLab Pages daemon constantly watches for modification time of `pages.path` - # It reloads configuration when `pages.path` is modified - update_file(pages_update_file, SecureRandom.hex(64)) - end - - def pages_path - @pages_path ||= project.pages_path - end - - def pages_config_file - File.join(pages_path, 'config.json') - end - - def pages_update_file - File.join(::Settings.pages.path, '.update') - end - - def update_file(file, data) - temp_file = "#{file}.#{SecureRandom.hex(16)}" - File.open(temp_file, 'w') do |f| - f.write(data) - end - FileUtils.move(temp_file, file, force: true) - ensure - # In case if the updating fails - FileUtils.remove(temp_file, force: true) - end - - def file_equals?(file, data) - existing_data = read_file(file) - data == existing_data.to_s - end - - def read_file(file) - File.open(file, 'r') do |f| - f.read - end - rescue StandardError - nil - end - end -end diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 898125c181c..f3ea0967a99 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -41,18 +41,49 @@ module Projects remote_mirror.update_start! # LFS objects must be sent first, or the push has dangling pointers - send_lfs_objects!(remote_mirror) + lfs_status = send_lfs_objects!(remote_mirror) response = remote_mirror.update_repository + failed, failure_message = failure_status(lfs_status, response, remote_mirror) + + # When the issue https://gitlab.com/gitlab-org/gitlab/-/issues/349262 is closed, + # we can move this block within failure_status. + if failed + remote_mirror.mark_as_failed!(failure_message) + else + remote_mirror.update_finish! + end + end + + def failure_status(lfs_status, response, remote_mirror) + message = '' + failed = false + lfs_sync_failed = false + + if lfs_status&.dig(:status) == :error + lfs_sync_failed = true + message += "Error synchronizing LFS files:" + message += "\n\n#{lfs_status[:message]}\n\n" + + failed = Feature.enabled?(:remote_mirror_fail_on_lfs, project, default_enabled: :yaml) + end if response.divergent_refs.any? - message = "Some refs have diverged and have not been updated on the remote:" + message += "Some refs have diverged and have not been updated on the remote:" message += "\n\n#{response.divergent_refs.join("\n")}" + failed = true + end - remote_mirror.mark_as_failed!(message) - else - remote_mirror.update_finish! + if message.present? + Gitlab::AppJsonLogger.info(message: "Error synching remote mirror", + project_id: project.id, + project_path: project.full_path, + remote_mirror_id: remote_mirror.id, + lfs_sync_failed: lfs_sync_failed, + divergent_ref_list: response.divergent_refs) end + + [failed, message] end def send_lfs_objects!(remote_mirror) diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index b34ecf06e52..fb810af3e6b 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -104,7 +104,6 @@ module Projects system_hook_service.execute_hooks_for(project, :update) end - update_pages_config if changing_pages_related_config? update_pending_builds if runners_settings_toggled? end @@ -112,10 +111,6 @@ module Projects AfterRenameService.new(project, path_before: project.path_before_last_save, full_path_before: project.full_path_before_last_save) end - def changing_pages_related_config? - changing_pages_https_only? || changing_pages_access_level? - end - def update_failed! model_errors = project.errors.full_messages.to_sentence error_message = model_errors.presence || s_('UpdateProject|Project could not be updated!') @@ -143,10 +138,6 @@ module Projects params.dig(:project_feature_attributes, :wiki_access_level).to_i > ProjectFeature::DISABLED end - def changing_pages_access_level? - params.dig(:project_feature_attributes, :pages_access_level) - end - def ensure_wiki_exists return if project.create_wiki @@ -154,16 +145,6 @@ module Projects Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki').increment end - def update_pages_config - return unless project.pages_deployed? - - PagesUpdateConfigurationWorker.perform_async(project.id) - end - - def changing_pages_https_only? - project.previous_changes.include?(:pages_https_only) - end - def changing_repository_storage? new_repository_storage = params[:repository_storage] diff --git a/app/services/resource_access_tokens/create_service.rb b/app/services/resource_access_tokens/create_service.rb index e0371e5d80f..28ea1ac8296 100644 --- a/app/services/resource_access_tokens/create_service.rb +++ b/app/services/resource_access_tokens/create_service.rb @@ -63,7 +63,7 @@ module ResourceAccessTokens name: params[:name] || "#{resource.name.to_s.humanize} bot", email: generate_email, username: generate_username, - user_type: "#{resource_type}_bot".to_sym, + user_type: :project_bot, skip_confirmation: true # Bot users should always have their emails confirmed. } end @@ -75,7 +75,8 @@ module ResourceAccessTokens end def generate_email - email_pattern = "#{resource_type}#{resource.id}_bot%s@example.com" + # Default emaildomain need to be reworked. See gitlab-org/gitlab#260305 + email_pattern = "#{resource_type}#{resource.id}_bot%s@noreply.#{Gitlab.config.gitlab.host}" uniquify.string(-> (n) { Kernel.sprintf(email_pattern, n) }) do |s| User.find_by_email(s) diff --git a/app/services/resource_access_tokens/revoke_service.rb b/app/services/resource_access_tokens/revoke_service.rb index 9543ea4b68d..2aaf4cc31d2 100644 --- a/app/services/resource_access_tokens/revoke_service.rb +++ b/app/services/resource_access_tokens/revoke_service.rb @@ -43,13 +43,9 @@ module ResourceAccessTokens def find_member strong_memoize(:member) do - if resource.is_a?(Project) - resource.project_member(bot_user) - elsif resource.is_a?(Group) - resource.group_member(bot_user) - else - false - end + next false unless resource.is_a?(Project) || resource.is_a?(Group) + + resource.member(bot_user) end end diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 171d52c328d..28e487aa24d 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -7,6 +7,8 @@ class SearchService DEFAULT_PER_PAGE = Gitlab::SearchResults::DEFAULT_PER_PAGE MAX_PER_PAGE = 200 + attr_reader :params + def initialize(current_user, params = {}) @current_user = current_user @params = Gitlab::Search::Params.new(params, detect_abuse: prevent_abusive_searches?) @@ -159,7 +161,7 @@ class SearchService end end - attr_reader :current_user, :params + attr_reader :current_user end SearchService.prepend_mod_with('SearchService') diff --git a/app/services/users/upsert_credit_card_validation_service.rb b/app/services/users/upsert_credit_card_validation_service.rb index 61cf598f178..7190c82bea3 100644 --- a/app/services/users/upsert_credit_card_validation_service.rb +++ b/app/services/users/upsert_credit_card_validation_service.rb @@ -2,8 +2,9 @@ module Users class UpsertCreditCardValidationService < BaseService - def initialize(params) + def initialize(params, user) @params = params.to_h.with_indifferent_access + @current_user = user end def execute @@ -18,6 +19,8 @@ module Users ::Users::CreditCardValidation.upsert(@params) + ::Users::UpdateService.new(current_user, user: current_user, requires_credit_card_verification: false).execute! + ServiceResponse.success(message: 'CreditCardValidation was set') rescue ActiveRecord::InvalidForeignKey, ActiveRecord::NotNullViolation => e ServiceResponse.error(message: "Could not set CreditCardValidation: #{e.message}") diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index 79bdf34392f..33e34ec41e2 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -26,7 +26,6 @@ class WebHookService end REQUEST_BODY_SIZE_LIMIT = 25.megabytes - GITLAB_EVENT_HEADER = 'X-Gitlab-Event' attr_accessor :hook, :data, :hook_name, :request_options attr_reader :uniqueness_token @@ -50,6 +49,10 @@ class WebHookService def execute return { status: :error, message: 'Hook disabled' } unless hook.executable? + log_recursion_limit if recursion_blocked? + + Gitlab::WebHooks::RecursionDetection.register!(hook) + start_time = Gitlab::Metrics::System.monotonic_time response = if parsed_url.userinfo.blank? @@ -95,6 +98,10 @@ class WebHookService Gitlab::ApplicationContext.with_context(hook.application_context) do break log_rate_limit if rate_limited? + log_recursion_limit if recursion_blocked? + + data[:_gitlab_recursion_detection_request_uuid] = Gitlab::WebHooks::RecursionDetection::UUID.instance.request_uuid + WebHookWorker.perform_async(hook.id, data, hook_name) end end @@ -108,7 +115,7 @@ class WebHookService def make_request(url, basic_auth = false) Gitlab::HTTP.post(url, body: Gitlab::Json::LimitedEncoder.encode(data, limit: REQUEST_BODY_SIZE_LIMIT), - headers: build_headers(hook_name), + headers: build_headers, verify: hook.enable_ssl_verification, basic_auth: basic_auth, **request_options) @@ -129,7 +136,7 @@ class WebHookService trigger: trigger, url: url, execution_duration: execution_duration, - request_headers: build_headers(hook_name), + request_headers: build_headers, request_data: request_data, response_headers: format_response_headers(response), response_body: safe_response_body(response), @@ -151,15 +158,16 @@ class WebHookService end end - def build_headers(hook_name) + def build_headers @headers ||= begin - { + headers = { 'Content-Type' => 'application/json', 'User-Agent' => "GitLab/#{Gitlab::VERSION}", - GITLAB_EVENT_HEADER => self.class.hook_to_event(hook_name) - }.tap do |hash| - hash['X-Gitlab-Token'] = Gitlab::Utils.remove_line_breaks(hook.token) if hook.token.present? - end + Gitlab::WebHooks::GITLAB_EVENT_HEADER => self.class.hook_to_event(hook_name) + } + + headers['X-Gitlab-Token'] = Gitlab::Utils.remove_line_breaks(hook.token) if hook.token.present? + headers.merge!(Gitlab::WebHooks::RecursionDetection.header(hook)) end end @@ -186,6 +194,10 @@ class WebHookService ) end + def recursion_blocked? + Gitlab::WebHooks::RecursionDetection.block?(hook) + end + def rate_limit @rate_limit ||= hook.rate_limit end @@ -199,4 +211,15 @@ class WebHookService **Gitlab::ApplicationContext.current ) end + + def log_recursion_limit + Gitlab::AuthLogger.error( + message: 'Webhook recursion detected and will be blocked in future', + hook_id: hook.id, + hook_type: hook.type, + hook_name: hook_name, + recursion_detection: ::Gitlab::WebHooks::RecursionDetection.to_log(hook), + **Gitlab::ApplicationContext.current + ) + end end diff --git a/app/services/work_items/build_service.rb b/app/services/work_items/build_service.rb new file mode 100644 index 00000000000..15a26f81d7c --- /dev/null +++ b/app/services/work_items/build_service.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module WorkItems + class BuildService < ::Issues::BuildService + private + + def model_klass + ::WorkItem + end + end +end diff --git a/app/services/work_items/create_service.rb b/app/services/work_items/create_service.rb new file mode 100644 index 00000000000..49f7b89158b --- /dev/null +++ b/app/services/work_items/create_service.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module WorkItems + class CreateService + def initialize(project:, current_user: nil, params: {}, spam_params:) + @create_service = ::Issues::CreateService.new( + project: project, + current_user: current_user, + params: params, + spam_params: spam_params, + build_service: ::WorkItems::BuildService.new(project: project, current_user: current_user, params: params) + ) + end + + def execute + @create_service.execute + end + end +end |