diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
| commit | 311b0269b4eb9839fa63f80c8d7a58f32b8138a0 (patch) | |
| tree | 07e7870bca8aed6d61fdcc810731c50d2c40af47 /app/services | |
| parent | 27909cef6c4170ed9205afa7426b8d3de47cbb0c (diff) | |
| download | gitlab-ce-311b0269b4eb9839fa63f80c8d7a58f32b8138a0.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-5-stable-eev14.5.0-rc42
Diffstat (limited to 'app/services')
103 files changed, 1513 insertions, 330 deletions
diff --git a/app/services/alert_management/process_prometheus_alert_service.rb b/app/services/alert_management/process_prometheus_alert_service.rb index 605ab7a1869..1b377a3d367 100644 --- a/app/services/alert_management/process_prometheus_alert_service.rb +++ b/app/services/alert_management/process_prometheus_alert_service.rb @@ -4,6 +4,7 @@ module AlertManagement class ProcessPrometheusAlertService extend ::Gitlab::Utils::Override include ::AlertManagement::AlertProcessing + include ::AlertManagement::Responses def initialize(project, payload) @project = project @@ -18,7 +19,7 @@ module AlertManagement complete_post_processing_tasks - ServiceResponse.success + success(alert) end private @@ -40,9 +41,5 @@ module AlertManagement def resolving_alert? incoming_payload.resolved? end - - def bad_request - ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) - end end end diff --git a/app/services/audit_event_service.rb b/app/services/audit_event_service.rb index 558798c830d..563d4a924fc 100644 --- a/app/services/audit_event_service.rb +++ b/app/services/audit_event_service.rb @@ -119,6 +119,10 @@ class AuditEventService event end + def stream_event_to_external_destinations(_event) + # Defined in EE + end + def log_authentication_event_to_database return unless Gitlab::Database.read_write? && authentication_event? @@ -130,6 +134,7 @@ class AuditEventService def save_or_track(event) event.save! + stream_event_to_external_destinations(event) rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, audit_event_type: event.class.to_s) end diff --git a/app/services/authorized_project_update/project_access_changed_service.rb b/app/services/authorized_project_update/project_access_changed_service.rb new file mode 100644 index 00000000000..62bf4ced1ae --- /dev/null +++ b/app/services/authorized_project_update/project_access_changed_service.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module AuthorizedProjectUpdate + class ProjectAccessChangedService + def initialize(project_ids) + @project_ids = Array.wrap(project_ids) + end + + def execute(blocking: true) + bulk_args = @project_ids.map { |id| [id] } + + if blocking + AuthorizedProjectUpdate::ProjectRecalculateWorker.bulk_perform_and_wait(bulk_args) + else + AuthorizedProjectUpdate::ProjectRecalculateWorker.bulk_perform_async(bulk_args) # rubocop:disable Scalability/BulkPerformWithContext + end + end + end +end diff --git a/app/services/award_emojis/base_service.rb b/app/services/award_emojis/base_service.rb index a677d03a221..626e26d63b5 100644 --- a/app/services/award_emojis/base_service.rb +++ b/app/services/award_emojis/base_service.rb @@ -14,7 +14,7 @@ module AwardEmojis private def normalize_name(name) - Gitlab::Emoji.normalize_emoji_name(name) + TanukiEmoji.find_by_alpha_code(name)&.name || name end # Provide more error state data than what BaseService allows. diff --git a/app/services/base_service.rb b/app/services/base_service.rb index 275ebcc7bcd..c7380768e32 100644 --- a/app/services/base_service.rb +++ b/app/services/base_service.rb @@ -6,13 +6,7 @@ # and existing service will use these one by one. # After all are migrated, we can remove this class. # -# New services should consider inheriting from: -# -# - BaseContainerService for services scoped by container (project or group) -# - BaseProjectService for services scoped to projects -# - BaseGroupService for services scoped to groups -# -# or, create a new base class and update this comment. +# For new services, please see https://docs.gitlab.com/ee/development/reusing_abstractions.html#service-classes class BaseService include BaseServiceUtility include Gitlab::Experiment::Dsl diff --git a/app/services/bulk_imports/file_download_service.rb b/app/services/bulk_imports/file_download_service.rb index 9a301c260a9..d08dc72e30b 100644 --- a/app/services/bulk_imports/file_download_service.rb +++ b/app/services/bulk_imports/file_download_service.rb @@ -7,8 +7,16 @@ module BulkImports REMOTE_FILENAME_PATTERN = %r{filename="(?<filename>[^"]+)"}.freeze FILENAME_SIZE_LIMIT = 255 # chars before the extension - - def initialize(configuration:, relative_url:, dir:, file_size_limit:, allowed_content_types:, filename: nil) + DEFAULT_FILE_SIZE_LIMIT = 5.gigabytes + DEFAULT_ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze + + def initialize( + configuration:, + relative_url:, + dir:, + file_size_limit: DEFAULT_FILE_SIZE_LIMIT, + allowed_content_types: DEFAULT_ALLOWED_CONTENT_TYPES, + filename: nil) @configuration = configuration @relative_url = relative_url @filename = filename diff --git a/app/services/bulk_update_integration_service.rb b/app/services/bulk_update_integration_service.rb index 45465ba3946..29c4d0cc220 100644 --- a/app/services/bulk_update_integration_service.rb +++ b/app/services/bulk_update_integration_service.rb @@ -12,7 +12,7 @@ class BulkUpdateIntegrationService Integration.where(id: batch_ids).update_all(integration_hash) if integration.data_fields_present? - integration.data_fields.class.where(service_id: batch_ids).update_all(data_fields_hash) + integration.data_fields.class.where(data_fields_foreign_key => batch_ids).update_all(data_fields_hash) end end end @@ -22,6 +22,11 @@ class BulkUpdateIntegrationService attr_reader :integration, :batch + # service_id or integration_id + def data_fields_foreign_key + integration.data_fields.class.reflections['integration'].foreign_key + end + def integration_hash integration.to_integration_hash.tap { |json| json['inherit_from_id'] = integration.inherit_from_id || integration.id } end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index ba9665555cc..540e8f7b970 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -25,6 +25,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Populate, Gitlab::Ci::Pipeline::Chain::StopDryRun, Gitlab::Ci::Pipeline::Chain::Create, + Gitlab::Ci::Pipeline::Chain::CreateCrossDatabaseAssociations, Gitlab::Ci::Pipeline::Chain::Limit::Activity, Gitlab::Ci::Pipeline::Chain::Limit::JobActivity, Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines, diff --git a/app/services/ci/destroy_pipeline_service.rb b/app/services/ci/destroy_pipeline_service.rb index 476c7523d60..6fbde5d291c 100644 --- a/app/services/ci/destroy_pipeline_service.rb +++ b/app/services/ci/destroy_pipeline_service.rb @@ -12,7 +12,9 @@ module Ci # Ci::Pipeline#destroy triggers `use_fast_destroy :job_artifacts` and # ci_builds has ON DELETE CASCADE to ci_pipelines. The pipeline, the builds, # job and pipeline artifacts all get destroyed here. - pipeline.reset.destroy! + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/345664') do + pipeline.reset.destroy! + end ServiceResponse.success(message: 'Pipeline not found') rescue ActiveRecord::RecordNotFound diff --git a/app/services/ci/external_pull_requests/create_pipeline_service.rb b/app/services/ci/external_pull_requests/create_pipeline_service.rb index dd93ca4708e..66127c94d35 100644 --- a/app/services/ci/external_pull_requests/create_pipeline_service.rb +++ b/app/services/ci/external_pull_requests/create_pipeline_service.rb @@ -16,14 +16,9 @@ module Ci private def create_pipeline_for(pull_request) - if ::Feature.enabled?(:ci_create_external_pr_pipeline_async, project, default_enabled: :yaml) - Ci::ExternalPullRequests::CreatePipelineWorker.perform_async( - project.id, current_user.id, pull_request.id - ) - else - Ci::CreatePipelineService.new(project, current_user, create_params(pull_request)) - .execute(:external_pull_request_event, external_pull_request: pull_request) - end + Ci::ExternalPullRequests::CreatePipelineWorker.perform_async( + project.id, current_user.id, pull_request.id + ) end def create_params(pull_request) diff --git a/app/services/ci/generate_kubeconfig_service.rb b/app/services/ci/generate_kubeconfig_service.rb new file mode 100644 index 00000000000..18f68c0ff09 --- /dev/null +++ b/app/services/ci/generate_kubeconfig_service.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Ci + class GenerateKubeconfigService + def initialize(build) + @build = build + @template = Gitlab::Kubernetes::Kubeconfig::Template.new + end + + def execute + template.add_cluster( + name: cluster_name, + url: Gitlab::Kas.tunnel_url + ) + + agents.each do |agent| + user = user_name(agent) + + template.add_user( + name: user, + token: agent_token(agent) + ) + + template.add_context( + name: context_name(agent), + cluster: cluster_name, + user: user + ) + end + + template + end + + private + + attr_reader :build, :template + + def agents + build.pipeline.authorized_cluster_agents + end + + def cluster_name + 'gitlab' + end + + def user_name(agent) + ['agent', agent.id].join(delimiter) + end + + def context_name(agent) + [agent.project.full_path, agent.name].join(delimiter) + end + + def agent_token(agent) + ['ci', agent.id, build.token].join(delimiter) + end + + def delimiter + ':' + end + end +end diff --git a/app/services/ci/job_artifacts/create_service.rb b/app/services/ci/job_artifacts/create_service.rb index 9fc7c3b4d40..7c67a2e175d 100644 --- a/app/services/ci/job_artifacts/create_service.rb +++ b/app/services/ci/job_artifacts/create_service.rb @@ -19,6 +19,7 @@ module Ci def initialize(job) @job = job @project = job.project + @pipeline = job.pipeline if ::Feature.enabled?(:ci_update_unlocked_job_artifacts, @project) end def authorize(artifact_type:, filesize: nil) @@ -53,7 +54,7 @@ module Ci private - attr_reader :job, :project + attr_reader :job, :project, :pipeline def validate_requirements(artifact_type:, filesize:) return too_large_error if too_large?(artifact_type, filesize) @@ -85,34 +86,38 @@ module Ci expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - artifact = Ci::JobArtifact.new( + artifact_attributes = { job_id: job.id, project: project, - file: artifacts_file, - file_type: params[:artifact_type], - file_format: params[:artifact_format], - file_sha256: artifacts_file.sha256, - expire_in: expire_in) + expire_in: expire_in + } + + artifact_attributes[:locked] = pipeline.locked if ::Feature.enabled?(:ci_update_unlocked_job_artifacts, project) + + artifact = Ci::JobArtifact.new( + artifact_attributes.merge( + file: artifacts_file, + file_type: params[:artifact_type], + file_format: params[:artifact_format], + file_sha256: artifacts_file.sha256 + ) + ) artifact_metadata = if metadata_file Ci::JobArtifact.new( - job_id: job.id, - project: project, - file: metadata_file, - file_type: :metadata, - file_format: :gzip, - file_sha256: metadata_file.sha256, - expire_in: expire_in) + artifact_attributes.merge( + file: metadata_file, + file_type: :metadata, + file_format: :gzip, + file_sha256: metadata_file.sha256 + ) + ) end [artifact, artifact_metadata] end def parse_artifact(artifact) - unless Feature.enabled?(:ci_synchronous_artifact_parsing, project, default_enabled: true) - return success - end - case artifact.file_type when 'dotenv' then parse_dotenv_artifact(artifact) else success diff --git a/app/services/ci/job_artifacts/destroy_all_expired_service.rb b/app/services/ci/job_artifacts/destroy_all_expired_service.rb index 3e9cc95d135..e4f65736a58 100644 --- a/app/services/ci/job_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/job_artifacts/destroy_all_expired_service.rb @@ -24,7 +24,11 @@ module Ci # which is scheduled every 7 minutes. def execute in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do - destroy_job_artifacts_with_slow_iteration(Time.current) + if ::Feature.enabled?(:ci_destroy_unlocked_job_artifacts) + destroy_unlocked_job_artifacts(Time.current) + else + destroy_job_artifacts_with_slow_iteration(Time.current) + end end @removed_artifacts_count @@ -32,13 +36,21 @@ module Ci private + def destroy_unlocked_job_artifacts(start_at) + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + artifacts = Ci::JobArtifact.expired_before(start_at).artifact_unlocked.limit(BATCH_SIZE) + service_response = destroy_batch(artifacts) + @removed_artifacts_count += service_response[:destroyed_artifacts_count] + end + end + def destroy_job_artifacts_with_slow_iteration(start_at) Ci::JobArtifact.expired_before(start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index| # For performance reasons, join with ci_pipelines after the batch is queried. # See: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47496 artifacts = relation.unlocked - service_response = destroy_batch_async(artifacts) + service_response = destroy_batch(artifacts) @removed_artifacts_count += service_response[:destroyed_artifacts_count] break if loop_timeout?(start_at) @@ -46,7 +58,7 @@ module Ci end end - def destroy_batch_async(artifacts) + def destroy_batch(artifacts) Ci::JobArtifacts::DestroyBatchService.new(artifacts).execute end diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb index 8536b88ccc0..866b40c32d8 100644 --- a/app/services/ci/job_artifacts/destroy_batch_service.rb +++ b/app/services/ci/job_artifacts/destroy_batch_service.rb @@ -26,15 +26,18 @@ module Ci def execute(update_stats: true) return success(destroyed_artifacts_count: 0, statistics_updates: {}) if @job_artifacts.empty? + destroy_related_records(@job_artifacts) + Ci::DeletedObject.transaction do Ci::DeletedObject.bulk_import(@job_artifacts, @pick_up_at) Ci::JobArtifact.id_in(@job_artifacts.map(&:id)).delete_all - destroy_related_records(@job_artifacts) end + after_batch_destroy_hook(@job_artifacts) + # This is executed outside of the transaction because it depends on Redis update_project_statistics! if update_stats - increment_monitoring_statistics(artifacts_count) + increment_monitoring_statistics(artifacts_count, artifacts_bytes) success(destroyed_artifacts_count: artifacts_count, statistics_updates: affected_project_statistics) @@ -43,9 +46,12 @@ module Ci private - # This method is implemented in EE and it must do only database work + # Overriden in EE def destroy_related_records(artifacts); end + # Overriden in EE + def after_batch_destroy_hook(artifacts); end + # using ! here since this can't be called inside a transaction def update_project_statistics! affected_project_statistics.each do |project, delta| @@ -63,8 +69,9 @@ module Ci end end - def increment_monitoring_statistics(size) - metrics.increment_destroyed_artifacts(size) + def increment_monitoring_statistics(size, bytes) + metrics.increment_destroyed_artifacts_count(size) + metrics.increment_destroyed_artifacts_bytes(bytes) end def metrics @@ -76,6 +83,12 @@ module Ci @job_artifacts.count end end + + def artifacts_bytes + strong_memoize(:artifacts_bytes) do + @job_artifacts.sum { |artifact| artifact.try(:size) || 0 } + end + end end end end diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb index 2ee9be476bb..725ecbcce5d 100644 --- a/app/services/ci/parse_dotenv_artifact_service.rb +++ b/app/services/ci/parse_dotenv_artifact_service.rb @@ -2,8 +2,7 @@ module Ci class ParseDotenvArtifactService < ::BaseService - MAX_ACCEPTABLE_DOTENV_SIZE = 5.kilobytes - MAX_ACCEPTABLE_VARIABLES_COUNT = 20 + include ::Gitlab::Utils::StrongMemoize SizeLimitError = Class.new(StandardError) ParserError = Class.new(StandardError) @@ -27,9 +26,9 @@ module Ci raise ArgumentError, 'Artifact is not dotenv file type' end - unless artifact.file.size < MAX_ACCEPTABLE_DOTENV_SIZE + unless artifact.file.size < dotenv_size_limit raise SizeLimitError, - "Dotenv Artifact Too Big. Maximum Allowable Size: #{MAX_ACCEPTABLE_DOTENV_SIZE}" + "Dotenv Artifact Too Big. Maximum Allowable Size: #{dotenv_size_limit}" end end @@ -45,9 +44,9 @@ module Ci end end - if variables.size > MAX_ACCEPTABLE_VARIABLES_COUNT + if variables.size > dotenv_variable_limit raise SizeLimitError, - "Dotenv files cannot have more than #{MAX_ACCEPTABLE_VARIABLES_COUNT} variables" + "Dotenv files cannot have more than #{dotenv_variable_limit} variables" end variables @@ -60,5 +59,13 @@ module Ci result.each(&:strip!) end + + def dotenv_variable_limit + strong_memoize(:dotenv_variable_limit) { project.actual_limits.dotenv_variables } + end + + def dotenv_size_limit + strong_memoize(:dotenv_size_limit) { project.actual_limits.dotenv_size } + end end end diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index 07cfbb9ce3c..ebb07de9d29 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -63,7 +63,7 @@ module Ci def clone_build(build) project.builds.new(build_attributes(build)).tap do |new_build| - new_build.assign_attributes(::Gitlab::Ci::Pipeline::Seed::Build.environment_attributes_for(new_build)) + new_build.assign_attributes(deployment_attributes_for(new_build, build)) end end @@ -75,6 +75,11 @@ module Ci attributes[:user] = current_user attributes end + + def deployment_attributes_for(new_build, old_build) + ::Gitlab::Ci::Pipeline::Seed::Build + .deployment_attributes_for(new_build, old_build.persisted_environment) + end end end diff --git a/app/services/ci/unlock_artifacts_service.rb b/app/services/ci/unlock_artifacts_service.rb index 7c169cb8395..30da31ba8ec 100644 --- a/app/services/ci/unlock_artifacts_service.rb +++ b/app/services/ci/unlock_artifacts_service.rb @@ -5,22 +5,84 @@ module Ci BATCH_SIZE = 100 def execute(ci_ref, before_pipeline = nil) - query = <<~SQL.squish - UPDATE "ci_pipelines" - SET "locked" = #{::Ci::Pipeline.lockeds[:unlocked]} - WHERE "ci_pipelines"."id" in ( - #{collect_pipelines(ci_ref, before_pipeline).select(:id).to_sql} - LIMIT #{BATCH_SIZE} - FOR UPDATE SKIP LOCKED - ) - RETURNING "ci_pipelines"."id"; - SQL - - loop do - break if Ci::Pipeline.connection.exec_query(query).empty? + results = { + unlocked_pipelines: 0, + unlocked_job_artifacts: 0 + } + + if ::Feature.enabled?(:ci_update_unlocked_job_artifacts, ci_ref.project) + loop do + unlocked_pipelines = [] + unlocked_job_artifacts = [] + + ::Ci::Pipeline.transaction do + unlocked_pipelines = unlock_pipelines(ci_ref, before_pipeline) + unlocked_job_artifacts = unlock_job_artifacts(unlocked_pipelines) + end + + break if unlocked_pipelines.empty? + + results[:unlocked_pipelines] += unlocked_pipelines.length + results[:unlocked_job_artifacts] += unlocked_job_artifacts.length + end + else + query = <<~SQL.squish + UPDATE "ci_pipelines" + SET "locked" = #{::Ci::Pipeline.lockeds[:unlocked]} + WHERE "ci_pipelines"."id" in ( + #{collect_pipelines(ci_ref, before_pipeline).select(:id).to_sql} + LIMIT #{BATCH_SIZE} + FOR UPDATE SKIP LOCKED + ) + RETURNING "ci_pipelines"."id"; + SQL + + loop do + unlocked_pipelines = Ci::Pipeline.connection.exec_query(query) + + break if unlocked_pipelines.empty? + + results[:unlocked_pipelines] += unlocked_pipelines.length + end end + + results end + # rubocop:disable CodeReuse/ActiveRecord + def unlock_job_artifacts_query(pipeline_ids) + ci_job_artifacts = ::Ci::JobArtifact.arel_table + + build_ids = ::Ci::Build.select(:id).where(commit_id: pipeline_ids) + + returning = Arel::Nodes::Grouping.new(ci_job_artifacts[:id]) + + Arel::UpdateManager.new + .table(ci_job_artifacts) + .where(ci_job_artifacts[:job_id].in(Arel.sql(build_ids.to_sql))) + .set([[ci_job_artifacts[:locked], ::Ci::JobArtifact.lockeds[:unlocked]]]) + .to_sql + " RETURNING #{returning.to_sql}" + end + # rubocop:enable CodeReuse/ActiveRecord + + # rubocop:disable CodeReuse/ActiveRecord + def unlock_pipelines_query(ci_ref, before_pipeline) + ci_pipelines = ::Ci::Pipeline.arel_table + + pipelines_scope = ci_ref.pipelines.artifacts_locked + pipelines_scope = pipelines_scope.before_pipeline(before_pipeline) if before_pipeline + pipelines_scope = pipelines_scope.select(:id).limit(BATCH_SIZE).lock('FOR UPDATE SKIP LOCKED') + + returning = Arel::Nodes::Grouping.new(ci_pipelines[:id]) + + Arel::UpdateManager.new + .table(ci_pipelines) + .where(ci_pipelines[:id].in(Arel.sql(pipelines_scope.to_sql))) + .set([[ci_pipelines[:locked], ::Ci::Pipeline.lockeds[:unlocked]]]) + .to_sql + " RETURNING #{returning.to_sql}" + end + # rubocop:enable CodeReuse/ActiveRecord + private def collect_pipelines(ci_ref, before_pipeline) @@ -29,5 +91,17 @@ module Ci pipeline_scope.artifacts_locked end + + def unlock_job_artifacts(pipelines) + return if pipelines.empty? + + ::Ci::JobArtifact.connection.exec_query( + unlock_job_artifacts_query(pipelines.rows.flatten) + ) + end + + def unlock_pipelines(ci_ref, before_pipeline) + ::Ci::Pipeline.connection.exec_query(unlock_pipelines_query(ci_ref, before_pipeline)) + end end end diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb index 3b403f92486..826d9a2eda3 100644 --- a/app/services/ci/update_build_state_service.rb +++ b/app/services/ci/update_build_state_service.rb @@ -73,11 +73,11 @@ module Ci ::Gitlab::Ci::Trace::Checksum.new(build).then do |checksum| unless checksum.valid? metrics.increment_trace_operation(operation: :invalid) - metrics.increment_error_counter(type: :chunks_invalid_checksum) + metrics.increment_error_counter(error_reason: :chunks_invalid_checksum) if checksum.corrupted? metrics.increment_trace_operation(operation: :corrupted) - metrics.increment_error_counter(type: :chunks_invalid_size) + metrics.increment_error_counter(error_reason: :chunks_invalid_size) end next unless log_invalid_chunks? diff --git a/app/services/clusters/agents/refresh_authorization_service.rb b/app/services/clusters/agents/refresh_authorization_service.rb index 7f401eef720..54b90a7304c 100644 --- a/app/services/clusters/agents/refresh_authorization_service.rb +++ b/app/services/clusters/agents/refresh_authorization_service.rb @@ -86,7 +86,7 @@ module Clusters if group_root_ancestor? root_ancestor.all_projects else - ::Project.none + ::Project.id_in(project.id) end end diff --git a/app/services/clusters/cleanup/project_namespace_service.rb b/app/services/clusters/cleanup/project_namespace_service.rb index 16254041306..0173f93f625 100644 --- a/app/services/clusters/cleanup/project_namespace_service.rb +++ b/app/services/clusters/cleanup/project_namespace_service.rb @@ -35,9 +35,11 @@ module Clusters end def kubeclient_delete_namespace(kubernetes_namespace) - cluster.kubeclient.delete_namespace(kubernetes_namespace.namespace) + cluster.kubeclient&.delete_namespace(kubernetes_namespace.namespace) rescue Kubeclient::ResourceNotFoundError - # no-op: nothing to delete + # The resources have already been deleted, possibly on a previous attempt that timed out + rescue Gitlab::UrlBlocker::BlockedUrlError + # User gave an invalid cluster from the start, or deleted the endpoint before this job ran end end end diff --git a/app/services/clusters/cleanup/service_account_service.rb b/app/services/clusters/cleanup/service_account_service.rb index baac9e4a9e7..53f968cd409 100644 --- a/app/services/clusters/cleanup/service_account_service.rb +++ b/app/services/clusters/cleanup/service_account_service.rb @@ -16,11 +16,14 @@ module Clusters def delete_gitlab_service_account log_event(:deleting_gitlab_service_account) - cluster.kubeclient.delete_service_account( + cluster.kubeclient&.delete_service_account( ::Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAME, ::Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE ) rescue Kubeclient::ResourceNotFoundError + # The resources have already been deleted, possibly on a previous attempt that timed out + rescue Gitlab::UrlBlocker::BlockedUrlError + # User gave an invalid cluster from the start, or deleted the endpoint before this job ran end end end diff --git a/app/services/clusters/applications/prometheus_health_check_service.rb b/app/services/clusters/integrations/prometheus_health_check_service.rb index eda47f56e72..cd06e59449c 100644 --- a/app/services/clusters/applications/prometheus_health_check_service.rb +++ b/app/services/clusters/integrations/prometheus_health_check_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Clusters - module Applications + module Integrations class PrometheusHealthCheckService include Gitlab::Utils::StrongMemoize include Gitlab::Routing @@ -14,7 +14,7 @@ module Clusters def execute raise 'Invalid cluster type. Only project types are allowed.' unless @cluster.project_type? - return unless prometheus_application.installed? + return unless prometheus_integration.enabled project = @cluster.clusterable @@ -28,32 +28,46 @@ module Clusters send_notification(project) if became_unhealthy? - prometheus_application.update_columns(healthy: currently_healthy?) if health_changed? + prometheus_integration.update_columns(health_status: current_health_status) if health_changed? end private - def prometheus_application - strong_memoize(:prometheus_application) do - @cluster.application_prometheus + def prometheus_integration + strong_memoize(:prometheus_integration) do + @cluster.integration_prometheus + end + end + + def current_health_status + if currently_healthy? + :healthy + else + :unhealthy end end def currently_healthy? strong_memoize(:currently_healthy) do - prometheus_application.prometheus_client.healthy? + prometheus_integration.prometheus_client.healthy? end end def became_unhealthy? strong_memoize(:became_unhealthy) do - (was_healthy? || was_healthy?.nil?) && !currently_healthy? + (was_healthy? || was_unknown?) && !currently_healthy? end end def was_healthy? strong_memoize(:was_healthy) do - prometheus_application.healthy + prometheus_integration.healthy? + end + end + + def was_unknown? + strong_memoize(:was_unknown) do + prometheus_integration.unknown? end end diff --git a/app/services/concerns/alert_management/responses.rb b/app/services/concerns/alert_management/responses.rb new file mode 100644 index 00000000000..183a831a00a --- /dev/null +++ b/app/services/concerns/alert_management/responses.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module AlertManagement + # Module to hold common response logic for AlertManagement services. + module Responses + def success(alerts) + ServiceResponse.success(payload: { alerts: Array(alerts) }) + end + + def bad_request + ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) + end + + def unauthorized + ServiceResponse.error(message: 'Unauthorized', http_status: :unauthorized) + end + + def unprocessable_entity + ServiceResponse.error(message: 'Unprocessable Entity', http_status: :unprocessable_entity) + end + + def forbidden + ServiceResponse.error(message: 'Forbidden', http_status: :forbidden) + end + end +end diff --git a/app/services/concerns/issues/issue_type_helpers.rb b/app/services/concerns/issues/issue_type_helpers.rb new file mode 100644 index 00000000000..44c20d20ff1 --- /dev/null +++ b/app/services/concerns/issues/issue_type_helpers.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Issues + module IssueTypeHelpers + # @param object [Issue, Project] + # @param issue_type [String, Symbol] + def create_issue_type_allowed?(object, issue_type) + WorkItem::Type.base_types.key?(issue_type.to_s) && + can?(current_user, :"create_#{issue_type}", object) + end + end +end diff --git a/app/services/concerns/members/bulk_create_users.rb b/app/services/concerns/members/bulk_create_users.rb index 4498f40c396..b98917f1396 100644 --- a/app/services/concerns/members/bulk_create_users.rb +++ b/app/services/concerns/members/bulk_create_users.rb @@ -6,7 +6,7 @@ module Members included do class << self - def add_users(source, users, access_level, current_user: nil, expires_at: nil) + def add_users(source, users, access_level, current_user: nil, expires_at: nil, tasks_to_be_done: [], tasks_project_id: nil) return [] unless users.present? emails, users, existing_members = parse_users_list(source, users) @@ -18,7 +18,9 @@ module Members access_level, existing_members: existing_members, current_user: current_user, - expires_at: expires_at) + expires_at: expires_at, + tasks_to_be_done: tasks_to_be_done, + tasks_project_id: tasks_project_id) .execute end end diff --git a/app/services/customer_relations/contacts/base_service.rb b/app/services/customer_relations/contacts/base_service.rb index 89f6f2c3f1f..1797e5021a1 100644 --- a/app/services/customer_relations/contacts/base_service.rb +++ b/app/services/customer_relations/contacts/base_service.rb @@ -6,7 +6,7 @@ module CustomerRelations private def allowed? - current_user&.can?(:admin_contact, group) + current_user&.can?(:admin_crm_contact, group) end def error(message) diff --git a/app/services/customer_relations/organizations/base_service.rb b/app/services/customer_relations/organizations/base_service.rb index 8f8480d697c..9b4ccafcea6 100644 --- a/app/services/customer_relations/organizations/base_service.rb +++ b/app/services/customer_relations/organizations/base_service.rb @@ -6,7 +6,7 @@ module CustomerRelations private def allowed? - current_user&.can?(:admin_organization, group) + current_user&.can?(:admin_crm_organization, group) end def error(message) diff --git a/app/services/dependency_proxy/find_or_create_blob_service.rb b/app/services/dependency_proxy/find_or_create_blob_service.rb index 0a6db6e3d34..1b43263a3ba 100644 --- a/app/services/dependency_proxy/find_or_create_blob_service.rb +++ b/app/services/dependency_proxy/find_or_create_blob_service.rb @@ -30,8 +30,7 @@ module DependencyProxy blob.save! end - # Technical debt: change to read_at https://gitlab.com/gitlab-org/gitlab/-/issues/341536 - blob.touch if from_cache + blob.read! if from_cache success(blob: blob, from_cache: from_cache) end diff --git a/app/services/dependency_proxy/find_or_create_manifest_service.rb b/app/services/dependency_proxy/find_or_create_manifest_service.rb index 1976d4d47f4..aeb62be9f3a 100644 --- a/app/services/dependency_proxy/find_or_create_manifest_service.rb +++ b/app/services/dependency_proxy/find_or_create_manifest_service.rb @@ -14,18 +14,18 @@ module DependencyProxy def execute @manifest = @group.dependency_proxy_manifests .active - .find_or_initialize_by_file_name_or_digest(file_name: @file_name, digest: @tag) + .find_by_file_name_or_digest(file_name: @file_name, digest: @tag) head_result = DependencyProxy::HeadManifestService.new(@image, @tag, @token).execute - if cached_manifest_matches?(head_result) - @manifest.touch + return respond if cached_manifest_matches?(head_result) - return success(manifest: @manifest, from_cache: true) + if Feature.enabled?(:dependency_proxy_manifest_workhorse, @group, default_enabled: :yaml) + success(manifest: nil, from_cache: false) + else + pull_new_manifest + respond(from_cache: false) end - - pull_new_manifest - respond(from_cache: false) rescue Timeout::Error, *Gitlab::HTTP::HTTP_ERRORS respond end @@ -34,12 +34,19 @@ module DependencyProxy def pull_new_manifest DependencyProxy::PullManifestService.new(@image, @tag, @token).execute_with_manifest do |new_manifest| - @manifest.update!( + params = { + file_name: @file_name, content_type: new_manifest[:content_type], digest: new_manifest[:digest], file: new_manifest[:file], size: new_manifest[:file].size - ) + } + + if @manifest + @manifest.update!(params) + else + @manifest = @group.dependency_proxy_manifests.create!(params) + end end end @@ -50,9 +57,8 @@ module DependencyProxy end def respond(from_cache: true) - if @manifest.persisted? - # Technical debt: change to read_at https://gitlab.com/gitlab-org/gitlab/-/issues/341536 - @manifest.touch if from_cache + if @manifest + @manifest.read! success(manifest: @manifest, from_cache: from_cache) else diff --git a/app/services/dependency_proxy/head_manifest_service.rb b/app/services/dependency_proxy/head_manifest_service.rb index ecc3eb77399..cd575b83a98 100644 --- a/app/services/dependency_proxy/head_manifest_service.rb +++ b/app/services/dependency_proxy/head_manifest_service.rb @@ -14,7 +14,10 @@ module DependencyProxy response = Gitlab::HTTP.head(manifest_url, headers: auth_headers.merge(Accept: ACCEPT_HEADERS)) if response.success? - success(digest: response.headers['docker-content-digest'], content_type: response.headers['content-type']) + success( + digest: response.headers[DependencyProxy::Manifest::DIGEST_HEADER], + content_type: response.headers['content-type'] + ) else error(response.body, response.code) end diff --git a/app/services/dependency_proxy/pull_manifest_service.rb b/app/services/dependency_proxy/pull_manifest_service.rb index 31494773cc0..e8f0ad6374a 100644 --- a/app/services/dependency_proxy/pull_manifest_service.rb +++ b/app/services/dependency_proxy/pull_manifest_service.rb @@ -20,7 +20,13 @@ module DependencyProxy file.write(response.body) file.flush - yield(success(file: file, digest: response.headers['docker-content-digest'], content_type: response.headers['content-type'])) + yield( + success( + file: file, + digest: response.headers[DependencyProxy::Manifest::DIGEST_HEADER], + content_type: response.headers['content-type'] + ) + ) ensure file.close file.unlink diff --git a/app/services/deployments/archive_in_project_service.rb b/app/services/deployments/archive_in_project_service.rb new file mode 100644 index 00000000000..a593721f390 --- /dev/null +++ b/app/services/deployments/archive_in_project_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Deployments + # This service archives old deploymets and deletes deployment refs for + # keeping the project repository performant. + class ArchiveInProjectService < ::BaseService + BATCH_SIZE = 100 + + def execute + unless ::Feature.enabled?(:deployments_archive, project, default_enabled: :yaml) + return error('Feature flag is not enabled') + end + + deployments = Deployment.archivables_in(project, limit: BATCH_SIZE) + + return success(result: :empty) if deployments.empty? + + ids = deployments.map(&:id) + ref_paths = deployments.map(&:ref_path) + + project.repository.delete_refs(*ref_paths) + project.deployments.id_in(ids).update_all(archived: true) + + success(result: :archived, count: ids.count) + end + end +end diff --git a/app/services/deployments/link_merge_requests_service.rb b/app/services/deployments/link_merge_requests_service.rb index 39fbef5dee2..40385418e48 100644 --- a/app/services/deployments/link_merge_requests_service.rb +++ b/app/services/deployments/link_merge_requests_service.rb @@ -16,7 +16,7 @@ module Deployments # Review apps have the environment type set (e.g. to `review`, though the # exact value may differ). We don't want to link merge requests to review # app deployments, as this is not useful. - return if deployment.environment.environment_type + return unless deployment.environment.should_link_to_merge_requests? # This service is triggered by a Sidekiq worker, which only runs when a # deployment is successful. We add an extra check here in case we ever diff --git a/app/services/design_management/copy_design_collection/copy_service.rb b/app/services/design_management/copy_design_collection/copy_service.rb index c43696442d2..5e557e9ea53 100644 --- a/app/services/design_management/copy_design_collection/copy_service.rb +++ b/app/services/design_management/copy_design_collection/copy_service.rb @@ -181,12 +181,12 @@ module DesignManagement ) end - # TODO Replace `Gitlab::Database.main.bulk_insert` with `BulkInsertSafe` + # TODO Replace `ApplicationRecord.legacy_bulk_insert` with `BulkInsertSafe` # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. # When this is fixed, we can remove the call to # `with_project_iid_supply` above, since the objects will be instantiated # and callbacks (including `ensure_project_iid!`) will fire. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert DesignManagement::Design.table_name, new_rows, return_ids: true @@ -207,9 +207,9 @@ module DesignManagement ) end - # TODO Replace `Gitlab::Database.main.bulk_insert` with `BulkInsertSafe` + # TODO Replace `ApplicationRecord.legacy_bulk_insert` with `BulkInsertSafe` # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert DesignManagement::Version.table_name, new_rows, return_ids: true @@ -239,7 +239,7 @@ module DesignManagement end # We cannot use `BulkInsertSafe` because of the uploader mounted in `Action`. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert DesignManagement::Action.table_name, new_rows ) @@ -278,7 +278,7 @@ module DesignManagement # We cannot use `BulkInsertSafe` due to the LfsObjectsProject#update_project_statistics # callback that fires after_commit. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert LfsObjectsProject.table_name, new_rows, on_conflict: :do_nothing # Upsert diff --git a/app/services/emails/destroy_service.rb b/app/services/emails/destroy_service.rb index d10833e66cb..d211c3470b2 100644 --- a/app/services/emails/destroy_service.rb +++ b/app/services/emails/destroy_service.rb @@ -3,6 +3,8 @@ module Emails class DestroyService < ::Emails::BaseService def execute(email) + raise StandardError, 'Cannot delete primary email' if email.user_primary_email? + email.destroy && update_secondary_emails!(email.email) end diff --git a/app/services/error_tracking/collect_error_service.rb b/app/services/error_tracking/collect_error_service.rb index 477453a693e..304e3898ee5 100644 --- a/app/services/error_tracking/collect_error_service.rb +++ b/app/services/error_tracking/collect_error_service.rb @@ -15,7 +15,7 @@ module ErrorTracking ) # The payload field contains all the data on error including stacktrace in jsonb. - # Together with occured_at these are 2 main attributes that we need to save here. + # Together with occurred_at these are 2 main attributes that we need to save here. error.events.create!( environment: event['environment'], description: exception['value'], @@ -28,7 +28,18 @@ module ErrorTracking private def event - params[:event] + @event ||= format_event(params[:event]) + end + + def format_event(event) + # Some SDK send exception payload as Array. For exmple Go lang SDK. + # We need to convert it to hash format we expect. + if event['exception'].is_a?(Array) + exception = event['exception'] + event['exception'] = { 'values' => exception } + end + + event end def exception diff --git a/app/services/google_cloud/service_accounts_service.rb b/app/services/google_cloud/service_accounts_service.rb new file mode 100644 index 00000000000..29ed69693b0 --- /dev/null +++ b/app/services/google_cloud/service_accounts_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module GoogleCloud + ## + # GCP keys used to store Google Cloud Service Accounts + GCP_KEYS = %w[GCP_PROJECT_ID GCP_SERVICE_ACCOUNT GCP_SERVICE_ACCOUNT_KEY].freeze + + ## + # This service deals with GCP Service Accounts in GitLab + + class ServiceAccountsService < ::BaseService + ## + # Find GCP Service Accounts in a GitLab project + # + # This method looks up GitLab project's CI vars + # and returns Google Cloud Service Accounts combinations + # aligning GitLab project and environment to GCP projects + + def find_for_project + group_vars_by_environment.map do |environment_scope, value| + { + environment: environment_scope, + gcp_project: value['GCP_PROJECT_ID'], + service_account_exists: value['GCP_SERVICE_ACCOUNT'].present?, + service_account_key_exists: value['GCP_SERVICE_ACCOUNT_KEY'].present? + } + end + end + + private + + def group_vars_by_environment + filtered_vars = @project.variables.filter { |variable| GCP_KEYS.include? variable.key } + filtered_vars.each_with_object({}) do |variable, grouped| + grouped[variable.environment_scope] ||= {} + grouped[variable.environment_scope][variable.key] = variable.value + end + end + end +end diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb index f900927793a..da3cebc2e6d 100644 --- a/app/services/groups/create_service.rb +++ b/app/services/groups/create_service.rb @@ -6,6 +6,7 @@ module Groups @current_user = user @params = params.dup @chat_team = @params.delete(:create_chat_team) + @create_event = @params.delete(:create_event) end def execute @@ -42,15 +43,26 @@ module Groups end end + after_create_hook + @group end private + attr_reader :create_event + def after_build_hook(group, params) # overridden in EE end + def after_create_hook + if group.persisted? && group.root? + delay = Namespaces::InviteTeamEmailService::DELIVERY_DELAY_IN_MINUTES + Namespaces::InviteTeamEmailWorker.perform_in(delay, group.id, current_user.id) + end + end + def remove_unallowed_params params.delete(:default_branch_protection) unless can?(current_user, :create_group_with_default_branch_protection) params.delete(:allow_mfa_for_subgroups) diff --git a/app/services/groups/import_export/import_service.rb b/app/services/groups/import_export/import_service.rb index f9db552f743..c8c2124078d 100644 --- a/app/services/groups/import_export/import_service.rb +++ b/app/services/groups/import_export/import_service.rb @@ -14,7 +14,7 @@ module Groups def async_execute group_import_state = GroupImportState.safe_find_or_create_by!(group: group, user: current_user) - jid = GroupImportWorker.perform_async(current_user.id, group.id) + jid = GroupImportWorker.with_status.perform_async(current_user.id, group.id) if jid.present? group_import_state.update!(jid: jid) diff --git a/app/services/groups/transfer_service.rb b/app/services/groups/transfer_service.rb index 334083a859f..cd89eb799dc 100644 --- a/app/services/groups/transfer_service.rb +++ b/app/services/groups/transfer_service.rb @@ -175,21 +175,18 @@ module Groups end def refresh_project_authorizations - ProjectAuthorization.where(project_id: @group.all_projects.select(:id)).delete_all # rubocop: disable CodeReuse/ActiveRecord + projects_to_update = Set.new - # refresh authorized projects for current_user immediately - current_user.refresh_authorized_projects - - # schedule refreshing projects for all the members of the group - @group.refresh_members_authorized_projects + # All projects in this hierarchy need to have their project authorizations recalculated + @group.all_projects.each_batch { |prjs| projects_to_update.merge(prjs.ids) } # rubocop: disable CodeReuse/ActiveRecord # When a group is transferred, it also affects who gets access to the projects shared to # the subgroups within its hierarchy, so we also schedule jobs that refresh authorizations for all such shared projects. - project_group_shares_within_the_hierarchy = ProjectGroupLink.in_group(group.self_and_descendants.select(:id)) - - project_group_shares_within_the_hierarchy.find_each do |project_group_link| - AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(project_group_link.project_id) + ProjectGroupLink.in_group(@group.self_and_descendants.select(:id)).each_batch do |project_group_links| + projects_to_update.merge(project_group_links.pluck(:project_id)) # rubocop: disable CodeReuse/ActiveRecord end + + AuthorizedProjectUpdate::ProjectAccessChangedService.new(projects_to_update.to_a).execute unless projects_to_update.empty? end def raise_transfer_error(message) @@ -199,7 +196,7 @@ module Groups def localized_error_messages { database_not_supported: s_('TransferGroup|Database is not supported.'), - namespace_with_same_path: s_('TransferGroup|The parent group already has a subgroup with the same path.'), + namespace_with_same_path: s_('TransferGroup|The parent group already has a subgroup or a project with the same path.'), group_is_already_root: s_('TransferGroup|Group is already a root group.'), same_parent_as_current: s_('TransferGroup|Group is already associated to the parent group.'), invalid_policies: s_("TransferGroup|You don't have enough permissions."), diff --git a/app/services/import/github/notes/create_service.rb b/app/services/import/github/notes/create_service.rb new file mode 100644 index 00000000000..79145f42313 --- /dev/null +++ b/app/services/import/github/notes/create_service.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Import + module Github + module Notes + class CreateService < ::Notes::CreateService + # Github does not have support to quick actions in notes (like /assign) + # Therefore, when importing notes we skip the quick actions processing + def quick_actions_supported?(_note) + false + end + end + end + end +end diff --git a/app/services/issuable/clone/attributes_rewriter.rb b/app/services/issuable/clone/attributes_rewriter.rb index d8b639bb422..279d3051848 100644 --- a/app/services/issuable/clone/attributes_rewriter.rb +++ b/app/services/issuable/clone/attributes_rewriter.rb @@ -99,7 +99,7 @@ module Issuable yield(event) end.compact - Gitlab::Database.main.bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert end end diff --git a/app/services/issuable_links/list_service.rb b/app/services/issuable_links/list_service.rb index fe9678dcc32..cc41a65379a 100644 --- a/app/services/issuable_links/list_service.rb +++ b/app/services/issuable_links/list_service.rb @@ -12,11 +12,16 @@ module IssuableLinks end def execute - serializer.new(current_user: current_user, issuable: issuable).represent(child_issuables) + serializer.new(current_user: current_user, issuable: issuable) + .represent(child_issuables, serializer_options) end private + def serializer_options + {} + end + def serializer raise NotImplementedError end diff --git a/app/services/issues/base_service.rb b/app/services/issues/base_service.rb index 6dce9fd6e73..efb5de5b17c 100644 --- a/app/services/issues/base_service.rb +++ b/app/services/issues/base_service.rb @@ -3,6 +3,7 @@ module Issues class BaseService < ::IssuableBaseService include IncidentManagement::UsageData + include IssueTypeHelpers def hook_data(issue, action, old_associations: {}) hook_data = issue.to_hook_data(current_user, old_associations: old_associations) @@ -44,7 +45,7 @@ module Issues def filter_params(issue) super - params.delete(:issue_type) unless issue_type_allowed?(issue) + params.delete(:issue_type) unless create_issue_type_allowed?(issue, params[:issue_type]) filter_incident_label(issue) if params[:issue_type] moved_issue = params.delete(:moved_issue) @@ -89,12 +90,6 @@ module Issues Milestones::IssuesCountService.new(milestone).delete_cache end - # @param object [Issue, Project] - def issue_type_allowed?(object) - WorkItem::Type.base_types.key?(params[:issue_type]) && - can?(current_user, :"create_#{params[:issue_type]}", object) - end - # @param issue [Issue] def filter_incident_label(issue) return unless add_incident_label?(issue) || remove_incident_label?(issue) diff --git a/app/services/issues/build_service.rb b/app/services/issues/build_service.rb index 7fdc8daf15c..8fd844c4886 100644 --- a/app/services/issues/build_service.rb +++ b/app/services/issues/build_service.rb @@ -80,7 +80,7 @@ module Issues ] allowed_params << :milestone_id if can?(current_user, :admin_issue, project) - allowed_params << :issue_type if issue_type_allowed?(project) + allowed_params << :issue_type if create_issue_type_allowed?(project, params[:issue_type]) params.slice(*allowed_params) end diff --git a/app/services/issues/close_service.rb b/app/services/issues/close_service.rb index ac846c769a3..65f143d0b21 100644 --- a/app/services/issues/close_service.rb +++ b/app/services/issues/close_service.rb @@ -62,6 +62,7 @@ module Issues def perform_incident_management_actions(issue) resolve_alert(issue) + resolve_incident(issue) end def close_external_issue(issue, closed_via) @@ -91,6 +92,14 @@ module Issues end end + def resolve_incident(issue) + return unless issue.incident? + + status = issue.incident_management_issuable_escalation_status || issue.build_incident_management_issuable_escalation_status + + SystemNoteService.resolve_incident_status(issue, current_user) if status.resolve + end + def store_first_mentioned_in_commit_at(issue, merge_request, max_commit_lookup: 100) metrics = issue.metrics return if metrics.nil? || metrics.first_mentioned_in_commit_at diff --git a/app/services/issues/create_service.rb b/app/services/issues/create_service.rb index fcedd1c1c8d..fa8d380404b 100644 --- a/app/services/issues/create_service.rb +++ b/app/services/issues/create_service.rb @@ -6,7 +6,7 @@ module Issues prepend RateLimitedService rate_limit key: :issues_create, - opts: { scope: [:project, :current_user], users_allowlist: -> { [User.support_bot.username] } } + opts: { scope: [:project, :current_user, :external_author] } # NOTE: For Issues::CreateService, we require the spam_params and do not default it to nil, because # spam_checking is likely to be necessary. However, if there is not a request available in scope @@ -25,6 +25,10 @@ module Issues create(@issue, skip_system_notes: skip_system_notes) end + def external_author + params[:external_author] # present when creating an issue using service desk (email: from) + end + def before_create(issue) Spam::SpamActionService.new( spammable: issue, diff --git a/app/services/issues/set_crm_contacts_service.rb b/app/services/issues/set_crm_contacts_service.rb new file mode 100644 index 00000000000..13fe30b5ac8 --- /dev/null +++ b/app/services/issues/set_crm_contacts_service.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Issues + class SetCrmContactsService < ::BaseProjectService + attr_accessor :issue, :errors + + MAX_ADDITIONAL_CONTACTS = 6 + + def execute(issue) + @issue = issue + @errors = [] + + return error_no_permissions unless allowed? + return error_invalid_params unless valid_params? + + determine_changes if params[:crm_contact_ids] + + return error_too_many if too_many? + + add_contacts if params[:add_crm_contact_ids] + remove_contacts if params[:remove_crm_contact_ids] + + if issue.valid? + ServiceResponse.success(payload: issue) + else + # The default error isn't very helpful: "Issue customer relations contacts is invalid" + issue.errors.delete(:issue_customer_relations_contacts) + issue.errors.add(:issue_customer_relations_contacts, errors.to_sentence) + ServiceResponse.error(payload: issue, message: issue.errors.full_messages) + end + end + + private + + def determine_changes + existing_contact_ids = issue.issue_customer_relations_contacts.map(&:contact_id) + params[:add_crm_contact_ids] = params[:crm_contact_ids] - existing_contact_ids + params[:remove_crm_contact_ids] = existing_contact_ids - params[:crm_contact_ids] + end + + def add_contacts + params[:add_crm_contact_ids].uniq.each do |contact_id| + issue_contact = issue.issue_customer_relations_contacts.create(contact_id: contact_id) + + unless issue_contact.persisted? + # The validation ensures that the id exists and the user has permission + errors << "#{contact_id}: The resource that you are attempting to access does not exist or you don't have permission to perform this action" + end + end + end + + def remove_contacts + issue.issue_customer_relations_contacts + .where(contact_id: params[:remove_crm_contact_ids]) # rubocop: disable CodeReuse/ActiveRecord + .delete_all + end + + def allowed? + current_user&.can?(:set_issue_crm_contacts, issue) + end + + def valid_params? + set_present? ^ add_or_remove_present? + end + + def set_present? + params[:crm_contact_ids].present? + end + + def add_or_remove_present? + params[:add_crm_contact_ids].present? || params[:remove_crm_contact_ids].present? + end + + def too_many? + params[:add_crm_contact_ids] && params[:add_crm_contact_ids].length > MAX_ADDITIONAL_CONTACTS + end + + def error_no_permissions + ServiceResponse.error(message: ['You have insufficient permissions to set customer relations contacts for this issue']) + end + + def error_invalid_params + ServiceResponse.error(message: ['You cannot combine crm_contact_ids with add_crm_contact_ids or remove_crm_contact_ids']) + end + + def error_too_many + ServiceResponse.error(payload: issue, message: ["You can only add up to #{MAX_ADDITIONAL_CONTACTS} contacts at one time"]) + end + end +end diff --git a/app/services/issues/update_service.rb b/app/services/issues/update_service.rb index d120b007af2..824a609dfb9 100644 --- a/app/services/issues/update_service.rb +++ b/app/services/issues/update_service.rb @@ -80,7 +80,7 @@ module Issues todo_service.reassigned_assignable(issue, current_user, old_assignees) track_incident_action(current_user, issue, :incident_assigned) - if Gitlab::ActionCable::Config.in_app? || Feature.enabled?(:broadcast_issue_updates, issue.project) + if Feature.enabled?(:broadcast_issue_updates, issue.project, default_enabled: :yaml) GraphqlTriggers.issuable_assignees_updated(issue) end end diff --git a/app/services/jira/requests/base.rb b/app/services/jira/requests/base.rb index 56484075d08..a16f8bbd367 100644 --- a/app/services/jira/requests/base.rb +++ b/app/services/jira/requests/base.rb @@ -67,9 +67,19 @@ module Jira ServiceResponse.error(message: error_message(e)) end + def auth_docs_link_start + auth_docs_link_url = Rails.application.routes.url_helpers.help_page_path('integration/jira', anchor: 'authentication-in-jira') + '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: auth_docs_link_url } + end + + def config_docs_link_start + config_docs_link_url = Rails.application.routes.url_helpers.help_page_path('integration/jira/configure') + '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: config_docs_link_url } + end + def error_message(error) reportable_error_message(error) || - s_('JiraRequest|An error occurred while requesting data from Jira. Check your Jira integration configuration and try again.') + s_('JiraRequest|An error occurred while requesting data from Jira. Check your %{docs_link_start}Jira integration configuration%{docs_link_end} and try again.').html_safe % { docs_link_start: config_docs_link_start, docs_link_end: '</a>'.html_safe } end # Returns a user-facing error message if possible, otherwise `nil`. @@ -93,11 +103,11 @@ module Jira def reportable_jira_ruby_error_message(error) case error.message when 'Unauthorized' - s_('JiraRequest|The credentials for accessing Jira are not valid. Check your Jira integration credentials and try again.') + s_('JiraRequest|The credentials for accessing Jira are not valid. Check your %{docs_link_start}Jira integration credentials%{docs_link_end} and try again.').html_safe % { docs_link_start: auth_docs_link_start, docs_link_end: '</a>'.html_safe } when 'Forbidden' - s_('JiraRequest|The credentials for accessing Jira are not allowed to access the data. Check your Jira integration credentials and try again.') + s_('JiraRequest|The credentials for accessing Jira are not allowed to access the data. Check your %{docs_link_start}Jira integration credentials%{docs_link_end} and try again.').html_safe % { docs_link_start: auth_docs_link_start, docs_link_end: '</a>'.html_safe } when 'Bad Request' - s_('JiraRequest|An error occurred while requesting data from Jira. Check your Jira integration configuration and try again.') + s_('JiraRequest|An error occurred while requesting data from Jira. Check your %{docs_link_start}Jira integration configuration%{docs_link_end} and try again.').html_safe % { docs_link_start: config_docs_link_start, docs_link_end: '</a>'.html_safe } when /errorMessages/ jira_ruby_json_error_message(error.message) end @@ -111,7 +121,7 @@ module Jira messages = Rails::Html::FullSanitizer.new.sanitize(messages).presence return unless messages - s_('JiraRequest|An error occurred while requesting data from Jira: %{messages}. Check your Jira integration configuration and try again.') % { messages: messages } + s_('JiraRequest|An error occurred while requesting data from Jira: %{messages}. Check your %{docs_link_start}Jira integration configuration%{docs_link_end} and try again.').html_safe % { messages: messages, docs_link_start: config_docs_link_start, docs_link_end: '</a>'.html_safe } rescue JSON::ParserError end end diff --git a/app/services/labels/transfer_service.rb b/app/services/labels/transfer_service.rb index a05090d6bfb..19d419609a5 100644 --- a/app/services/labels/transfer_service.rb +++ b/app/services/labels/transfer_service.rb @@ -50,21 +50,32 @@ module Labels # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_issues - @group_labels_applied_to_issues ||= Label.joins(:issues) - .where( - issues: { project_id: project.id }, - labels: { group_id: old_group.self_and_ancestors } - ) + @labels_applied_to_issues ||= if use_optimized_group_labels_query? + Label.joins(:issues) + .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) + .where(issues: { project_id: project.id }).reorder(nil) + else + Label.joins(:issues).where( + issues: { project_id: project.id }, + labels: { group_id: old_group.self_and_ancestors } + ) + end end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_merge_requests - @group_labels_applied_to_merge_requests ||= Label.joins(:merge_requests) - .where( - merge_requests: { target_project_id: project.id }, - labels: { group_id: old_group.self_and_ancestors } - ) + @labels_applied_to_mrs ||= if use_optimized_group_labels_query? + Label.joins(:merge_requests) + .joins("INNER JOIN namespaces on namespaces.id = labels.group_id AND namespaces.type = 'Group'" ) + .where(merge_requests: { target_project_id: project.id }).reorder(nil) + else + Label.joins(:merge_requests) + .where( + merge_requests: { target_project_id: project.id }, + labels: { group_id: old_group.self_and_ancestors } + ) + end end # rubocop: enable CodeReuse/ActiveRecord @@ -88,5 +99,9 @@ module Labels .update_all(label_id: new_label_id) end # rubocop: enable CodeReuse/ActiveRecord + + def use_optimized_group_labels_query? + Feature.enabled?(:use_optimized_group_labels_query, project.root_namespace, default_enabled: :yaml) + end end end diff --git a/app/services/loose_foreign_keys/batch_cleaner_service.rb b/app/services/loose_foreign_keys/batch_cleaner_service.rb new file mode 100644 index 00000000000..06c05e8ff54 --- /dev/null +++ b/app/services/loose_foreign_keys/batch_cleaner_service.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module LooseForeignKeys + class BatchCleanerService + def initialize(parent_klass:, deleted_parent_records:, modification_tracker: LooseForeignKeys::ModificationTracker.new, models_by_table_name:) + @parent_klass = parent_klass + @deleted_parent_records = deleted_parent_records + @modification_tracker = modification_tracker + @models_by_table_name = models_by_table_name + @deleted_records_counter = Gitlab::Metrics.counter( + :loose_foreign_key_processed_deleted_records, + 'The number of processed loose foreign key deleted records' + ) + end + + def execute + parent_klass.loose_foreign_key_definitions.each do |foreign_key_definition| + run_cleaner_service(foreign_key_definition, with_skip_locked: true) + break if modification_tracker.over_limit? + + run_cleaner_service(foreign_key_definition, with_skip_locked: false) + break if modification_tracker.over_limit? + end + + return if modification_tracker.over_limit? + + # At this point, all associations are cleaned up, we can update the status of the parent records + update_count = LooseForeignKeys::DeletedRecord.mark_records_processed(deleted_parent_records) + + deleted_records_counter.increment({ table: parent_klass.table_name, db_config_name: LooseForeignKeys::DeletedRecord.connection.pool.db_config.name }, update_count) + end + + private + + attr_reader :parent_klass, :deleted_parent_records, :modification_tracker, :models_by_table_name, :deleted_records_counter + + def record_result(cleaner, result) + if cleaner.async_delete? + modification_tracker.add_deletions(result[:table], result[:affected_rows]) + elsif cleaner.async_nullify? + modification_tracker.add_updates(result[:table], result[:affected_rows]) + end + end + + def run_cleaner_service(foreign_key_definition, with_skip_locked:) + cleaner = CleanerService.new( + model: models_by_table_name.fetch(foreign_key_definition.to_table), + foreign_key_definition: foreign_key_definition, + deleted_parent_records: deleted_parent_records, + with_skip_locked: with_skip_locked + ) + + loop do + result = cleaner.execute + record_result(cleaner, result) + + break if modification_tracker.over_limit? || result[:affected_rows] == 0 + end + end + end +end diff --git a/app/services/loose_foreign_keys/cleaner_service.rb b/app/services/loose_foreign_keys/cleaner_service.rb new file mode 100644 index 00000000000..8fe053e2edf --- /dev/null +++ b/app/services/loose_foreign_keys/cleaner_service.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module LooseForeignKeys + # rubocop: disable CodeReuse/ActiveRecord + class CleanerService + DELETE_LIMIT = 1000 + UPDATE_LIMIT = 500 + + delegate :connection, to: :model + + def initialize(model:, foreign_key_definition:, deleted_parent_records:, with_skip_locked: false) + @model = model + @foreign_key_definition = foreign_key_definition + @deleted_parent_records = deleted_parent_records + @with_skip_locked = with_skip_locked + end + + def execute + result = connection.execute(build_query) + + { affected_rows: result.cmd_tuples, table: foreign_key_definition.to_table } + end + + def async_delete? + foreign_key_definition.on_delete == :async_delete + end + + def async_nullify? + foreign_key_definition.on_delete == :async_nullify + end + + private + + attr_reader :model, :foreign_key_definition, :deleted_parent_records, :with_skip_locked + + def build_query + query = if async_delete? + delete_query + elsif async_nullify? + update_query + else + raise "Invalid on_delete argument: #{foreign_key_definition.on_delete}" + end + + unless query.include?(%{"#{foreign_key_definition.column}" IN (}) + raise("FATAL: foreign key condition is missing from the generated query: #{query}") + end + + query + end + + def arel_table + @arel_table ||= model.arel_table + end + + def primary_keys + @primary_keys ||= connection.primary_keys(model.table_name).map { |key| arel_table[key] } + end + + def quoted_table_name + @quoted_table_name ||= Arel.sql(connection.quote_table_name(model.table_name)) + end + + def delete_query + query = Arel::DeleteManager.new + query.from(quoted_table_name) + + add_in_query_with_limit(query, DELETE_LIMIT) + end + + def update_query + query = Arel::UpdateManager.new + query.table(quoted_table_name) + query.set([[arel_table[foreign_key_definition.column], nil]]) + + add_in_query_with_limit(query, UPDATE_LIMIT) + end + + # IN query with one or composite primary key + # WHERE (primary_key1, primary_key2) IN (subselect) + def add_in_query_with_limit(query, limit) + columns = Arel::Nodes::Grouping.new(primary_keys) + query.where(columns.in(in_query_with_limit(limit))).to_sql + end + + # Builds the following sub-query + # SELECT primary_keys FROM table WHERE foreign_key IN (1, 2, 3) LIMIT N + def in_query_with_limit(limit) + in_query = Arel::SelectManager.new + in_query.from(quoted_table_name) + in_query.where(arel_table[foreign_key_definition.column].in(deleted_parent_records.map(&:primary_key_value))) + in_query.projections = primary_keys + in_query.take(limit) + in_query.lock(Arel.sql('FOR UPDATE SKIP LOCKED')) if with_skip_locked + in_query + end + end + # rubocop: enable CodeReuse/ActiveRecord +end diff --git a/app/services/loose_foreign_keys/process_deleted_records_service.rb b/app/services/loose_foreign_keys/process_deleted_records_service.rb new file mode 100644 index 00000000000..735fc8a2415 --- /dev/null +++ b/app/services/loose_foreign_keys/process_deleted_records_service.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module LooseForeignKeys + class ProcessDeletedRecordsService + BATCH_SIZE = 1000 + + def initialize(connection:) + @connection = connection + end + + def execute + modification_tracker = ModificationTracker.new + + tracked_tables.cycle do |table| + records = load_batch_for_table(table) + + if records.empty? + tracked_tables.delete(table) + next + end + + break if modification_tracker.over_limit? + + model = find_parent_model!(table) + + LooseForeignKeys::BatchCleanerService + .new(parent_klass: model, + deleted_parent_records: records, + modification_tracker: modification_tracker, + models_by_table_name: models_by_table_name) + .execute + + break if modification_tracker.over_limit? + end + + modification_tracker.stats + end + + private + + attr_reader :connection + + def load_batch_for_table(table) + fully_qualified_table_name = "#{current_schema}.#{table}" + LooseForeignKeys::DeletedRecord.load_batch_for_table(fully_qualified_table_name, BATCH_SIZE) + end + + def find_parent_model!(table) + models_by_table_name.fetch(table) + end + + def current_schema + @current_schema = connection.current_schema + end + + def tracked_tables + @tracked_tables ||= models_by_table_name + .select { |table_name, model| model.respond_to?(:loose_foreign_key_definitions) } + .keys + end + + def models_by_table_name + @models_by_table_name ||= begin + all_models + .select(&:base_class?) + .index_by(&:table_name) + end + end + + def all_models + ApplicationRecord.descendants + end + end +end diff --git a/app/services/members/create_service.rb b/app/services/members/create_service.rb index 0cc62e661a3..cb905e01613 100644 --- a/app/services/members/create_service.rb +++ b/app/services/members/create_service.rb @@ -63,10 +63,14 @@ module Members invites, params[:access_level], expires_at: params[:expires_at], - current_user: current_user + current_user: current_user, + tasks_to_be_done: params[:tasks_to_be_done], + tasks_project_id: params[:tasks_project_id] ) members.each { |member| process_result(member) } + + create_tasks_to_be_done end def process_result(member) @@ -112,6 +116,19 @@ module Members end end + def create_tasks_to_be_done + return unless experiment(:invite_members_for_task).enabled? + return if params[:tasks_to_be_done].blank? || params[:tasks_project_id].blank? + + valid_members = members.select { |member| member.valid? && member.member_task.valid? } + return unless valid_members.present? + + # We can take the first `member_task` here, since all tasks will have the same attributes needed + # for the `TasksToBeDone::CreateWorker`, ie. `project` and `tasks_to_be_done`. + member_task = valid_members[0].member_task + TasksToBeDone::CreateWorker.perform_async(member_task.id, current_user.id, valid_members.map(&:user_id)) + end + def areas_of_focus params[:areas_of_focus] || [] end diff --git a/app/services/members/creator_service.rb b/app/services/members/creator_service.rb index 7b0bebff760..f2c8a6f20a1 100644 --- a/app/services/members/creator_service.rb +++ b/app/services/members/creator_service.rb @@ -4,6 +4,8 @@ module Members # This class serves as more of an app-wide way we add/create members # All roads to add members should take this path. class CreatorService + include Gitlab::Experiment::Dsl + class << self def parsed_access_level(access_level) access_levels.fetch(access_level) { access_level.to_i } @@ -24,6 +26,7 @@ module Members def execute find_or_build_member update_member + create_member_task member end @@ -61,6 +64,21 @@ module Members } end + def create_member_task + return unless experiment(:invite_members_for_task).enabled? + return unless member.persisted? + return if member_task_attributes.value?(nil) + + member.create_member_task(member_task_attributes) + end + + def member_task_attributes + { + tasks_to_be_done: args[:tasks_to_be_done], + project_id: args[:tasks_project_id] + } + end + def approve_request ::Members::ApproveAccessRequestService.new(current_user, access_level: access_level) diff --git a/app/services/members/invite_service.rb b/app/services/members/invite_service.rb index 257a986b8dd..85acb720f0f 100644 --- a/app/services/members/invite_service.rb +++ b/app/services/members/invite_service.rb @@ -39,6 +39,11 @@ module Members errors[invite_email(member)] = member.errors.full_messages.to_sentence end + override :create_tasks_to_be_done + def create_tasks_to_be_done + # Only create task issues for existing users. Tasks for new users are created when they signup. + end + def invite_email(member) member.invite_email || member.user.email end diff --git a/app/services/merge_requests/outdated_discussion_diff_lines_service.rb b/app/services/merge_requests/outdated_discussion_diff_lines_service.rb new file mode 100644 index 00000000000..a2de5a32963 --- /dev/null +++ b/app/services/merge_requests/outdated_discussion_diff_lines_service.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module MergeRequests + class OutdatedDiscussionDiffLinesService + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :note + + OVERFLOW_LINES_COUNT = 2 + + def initialize(project:, note:) + @project = project + @note = note + end + + def execute + end_position = position.line_range["end"] + diff_line_index = diff_lines.find_index do |l| + if end_position["new_line"] + l.new_line == end_position["new_line"] + elsif end_position["old_line"] + l.old_line == end_position["old_line"] + end + end + initial_line_index = [diff_line_index - OVERFLOW_LINES_COUNT, 0].max + last_line_index = [diff_line_index + OVERFLOW_LINES_COUNT, diff_lines.length].min + + prev_lines = [] + + diff_lines[initial_line_index..last_line_index].each do |line| + if line.meta? + prev_lines.clear + else + prev_lines << line + end + end + + prev_lines + end + + private + + def position + note.change_position + end + + def repository + project.repository + end + + def diff_file + position.diff_file(repository) + end + + def diff_lines + strong_memoize(:diff_lines) do + diff_file.highlighted_diff_lines + end + end + end +end diff --git a/app/services/merge_requests/retarget_chain_service.rb b/app/services/merge_requests/retarget_chain_service.rb index dab6e198979..33aae4184ae 100644 --- a/app/services/merge_requests/retarget_chain_service.rb +++ b/app/services/merge_requests/retarget_chain_service.rb @@ -5,8 +5,6 @@ module MergeRequests MAX_RETARGET_MERGE_REQUESTS = 4 def execute(merge_request) - return unless Feature.enabled?(:retarget_merge_requests, merge_request.target_project, default_enabled: :yaml) - # we can only retarget MRs that are targeting the same project return unless merge_request.for_same_project? && merge_request.merged? diff --git a/app/services/merge_requests/toggle_attention_requested_service.rb b/app/services/merge_requests/toggle_attention_requested_service.rb new file mode 100644 index 00000000000..66c5d6fce5d --- /dev/null +++ b/app/services/merge_requests/toggle_attention_requested_service.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module MergeRequests + class ToggleAttentionRequestedService < MergeRequests::BaseService + attr_accessor :merge_request, :user + + def initialize(project:, current_user:, merge_request:, user:) + super(project: project, current_user: current_user) + + @merge_request = merge_request + @user = user + end + + def execute + return error("Invalid permissions") unless can?(current_user, :update_merge_request, merge_request) + + if reviewer || assignee + update_state(reviewer) + update_state(assignee) + + if reviewer&.attention_requested? || assignee&.attention_requested? + notity_user + end + + success + else + error("User is not a reviewer or assignee of the merge request") + end + end + + private + + def notity_user + todo_service.create_attention_requested_todo(merge_request, current_user, user) + end + + def assignee + merge_request.find_assignee(user) + end + + def reviewer + merge_request.find_reviewer(user) + end + + def update_state(reviewer_or_assignee) + reviewer_or_assignee&.update(state: reviewer_or_assignee&.attention_requested? ? :reviewed : :attention_requested) + end + end +end diff --git a/app/services/namespaces/in_product_marketing_email_records.rb b/app/services/namespaces/in_product_marketing_email_records.rb new file mode 100644 index 00000000000..1237a05ea13 --- /dev/null +++ b/app/services/namespaces/in_product_marketing_email_records.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Namespaces + class InProductMarketingEmailRecords + attr_reader :records + + def initialize + @records = [] + end + + def save! + Users::InProductMarketingEmail.bulk_insert!(@records) + @records = [] + end + + def add(user, track, series) + @records << Users::InProductMarketingEmail.new( + user: user, + track: track, + series: series, + created_at: Time.zone.now, + updated_at: Time.zone.now + ) + end + end +end diff --git a/app/services/namespaces/in_product_marketing_emails_service.rb b/app/services/namespaces/in_product_marketing_emails_service.rb index 0401653cf3c..90900698e1a 100644 --- a/app/services/namespaces/in_product_marketing_emails_service.rb +++ b/app/services/namespaces/in_product_marketing_emails_service.rb @@ -56,7 +56,7 @@ module Namespaces def initialize(track, interval) @track = track @interval = interval - @in_product_marketing_email_records = [] + @sent_email_records = InProductMarketingEmailRecords.new end def execute @@ -71,17 +71,21 @@ module Namespaces private - attr_reader :track, :interval, :in_product_marketing_email_records + attr_reader :track, :interval, :sent_email_records + + def send_email(user, group) + NotificationService.new.in_product_marketing(user.id, group.id, track, series) + end def send_email_for_group(group) users_for_group(group).each do |user| if can_perform_action?(user, group) send_email(user, group) - track_sent_email(user, track, series) + sent_email_records.add(user, track, series) end end - save_tracked_emails! + sent_email_records.save! end def groups_for_track @@ -126,10 +130,6 @@ module Namespaces end end - def send_email(user, group) - NotificationService.new.in_product_marketing(user.id, group.id, track, series) - end - def completed_actions TRACKS[track][:completed_actions] end @@ -146,21 +146,6 @@ module Namespaces def series TRACKS[track][:interval_days].index(interval) end - - def save_tracked_emails! - Users::InProductMarketingEmail.bulk_insert!(in_product_marketing_email_records) - @in_product_marketing_email_records = [] - end - - def track_sent_email(user, track, series) - in_product_marketing_email_records << Users::InProductMarketingEmail.new( - user: user, - track: track, - series: series, - created_at: Time.zone.now, - updated_at: Time.zone.now - ) - end end end diff --git a/app/services/namespaces/invite_team_email_service.rb b/app/services/namespaces/invite_team_email_service.rb new file mode 100644 index 00000000000..45975d1953a --- /dev/null +++ b/app/services/namespaces/invite_team_email_service.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Namespaces + class InviteTeamEmailService + include Gitlab::Experiment::Dsl + + TRACK = :invite_team + DELIVERY_DELAY_IN_MINUTES = 20.minutes + + def self.send_email(user, group) + new(user, group).execute + end + + def initialize(user, group) + @group = group + @user = user + @sent_email_records = InProductMarketingEmailRecords.new + end + + def execute + return unless user.email_opted_in? + return unless group.root? + return unless group.setup_for_company + + # Exclude group if users other than the creator have already been + # added/invited + return unless group.member_count == 1 + + return if email_for_track_sent_to_user? + + experiment(:invite_team_email, group: group) do |e| + e.candidate do + send_email(user, group) + sent_email_records.add(user, track, series) + sent_email_records.save! + end + + e.record! + end + end + + private + + attr_reader :user, :group, :sent_email_records + + def send_email(user, group) + NotificationService.new.in_product_marketing(user.id, group.id, track, series) + end + + def track + TRACK + end + + def series + 0 + end + + def email_for_track_sent_to_user? + Users::InProductMarketingEmail.for_user_with_track_and_series(user, track, series).present? + end + end +end diff --git a/app/services/notes/create_service.rb b/app/services/notes/create_service.rb index 194c3d7bf7b..9a0db3bb9aa 100644 --- a/app/services/notes/create_service.rb +++ b/app/services/notes/create_service.rb @@ -43,7 +43,7 @@ module Notes private def execute_quick_actions(note) - return yield(false) unless quick_actions_service.supported?(note) + return yield(false) unless quick_actions_supported?(note) content, update_params, message = quick_actions_service.execute(note, quick_action_options) only_commands = content.empty? @@ -54,6 +54,10 @@ module Notes do_commands(note, update_params, message, only_commands) end + def quick_actions_supported?(note) + quick_actions_service.supported?(note) + end + def quick_actions_service @quick_actions_service ||= QuickActionsService.new(project, current_user) end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index afc9015e758..6ad3a74b85d 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -598,8 +598,8 @@ class NotificationService user.notification_email_for(pipeline.project.group) end - if recipients.any? - mailer.public_send(email_template, pipeline, recipients).deliver_later + recipients.each do |recipient| + mailer.public_send(email_template, pipeline, recipient).deliver_later end end diff --git a/app/services/packages/create_dependency_service.rb b/app/services/packages/create_dependency_service.rb index 2c80ec66dbc..10a86e44cb0 100644 --- a/app/services/packages/create_dependency_service.rb +++ b/app/services/packages/create_dependency_service.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true module Packages + # rubocop: disable Gitlab/BulkInsert class CreateDependencyService < BaseService attr_reader :package, :dependencies @@ -51,7 +52,7 @@ module Packages } end - ids = database.bulk_insert(Packages::Dependency.table_name, rows, return_ids: true, on_conflict: :do_nothing) + ids = ApplicationRecord.legacy_bulk_insert(Packages::Dependency.table_name, rows, return_ids: true, on_conflict: :do_nothing) return ids if ids.size == names_and_version_patterns.size Packages::Dependency.uncached do @@ -72,11 +73,8 @@ module Packages } end - database.bulk_insert(Packages::DependencyLink.table_name, rows) - end - - def database - ::Gitlab::Database.main + ApplicationRecord.legacy_bulk_insert(Packages::DependencyLink.table_name, rows) end end + # rubocop: enable Gitlab/BulkInsert end diff --git a/app/services/packages/npm/create_package_service.rb b/app/services/packages/npm/create_package_service.rb index 1d5d9c38432..ae9c92a3d3a 100644 --- a/app/services/packages/npm/create_package_service.rb +++ b/app/services/packages/npm/create_package_service.rb @@ -4,6 +4,8 @@ module Packages class CreatePackageService < ::Packages::CreatePackageService include Gitlab::Utils::StrongMemoize + PACKAGE_JSON_NOT_ALLOWED_FIELDS = %w[readme readmeFilename].freeze + def execute return error('Version is empty.', 400) if version.blank? return error('Package already exists.', 403) if current_package_exists? @@ -21,6 +23,10 @@ module Packages ::Packages::CreateDependencyService.new(package, package_dependencies).execute ::Packages::Npm::CreateTagService.new(package, dist_tag).execute + if Feature.enabled?(:packages_npm_abbreviated_metadata, project, default_enabled: :yaml) + package.create_npm_metadatum!(package_json: package_json) + end + package end @@ -46,6 +52,10 @@ module Packages params[:versions][version] end + def package_json + version_data.except(*PACKAGE_JSON_NOT_ALLOWED_FIELDS) + end + def dist_tag params['dist-tags'].each_key.first end diff --git a/app/services/packages/nuget/create_dependency_service.rb b/app/services/packages/nuget/create_dependency_service.rb index 3fc42056d43..85f295ac7b7 100644 --- a/app/services/packages/nuget/create_dependency_service.rb +++ b/app/services/packages/nuget/create_dependency_service.rb @@ -41,7 +41,7 @@ module Packages } end - ::Gitlab::Database.main.bulk_insert(::Packages::Nuget::DependencyLinkMetadatum.table_name, rows.compact) # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert(::Packages::Nuget::DependencyLinkMetadatum.table_name, rows.compact) # rubocop:disable Gitlab/BulkInsert end def raw_dependency_for(dependency) diff --git a/app/services/packages/rubygems/create_dependencies_service.rb b/app/services/packages/rubygems/create_dependencies_service.rb index dea429148cf..0b2ae56bf45 100644 --- a/app/services/packages/rubygems/create_dependencies_service.rb +++ b/app/services/packages/rubygems/create_dependencies_service.rb @@ -3,8 +3,6 @@ module Packages module Rubygems class CreateDependenciesService - include BulkInsertSafe - def initialize(package, gemspec) @package = package @gemspec = gemspec diff --git a/app/services/packages/update_tags_service.rb b/app/services/packages/update_tags_service.rb index 2bdf75a6617..f29c54dacb9 100644 --- a/app/services/packages/update_tags_service.rb +++ b/app/services/packages/update_tags_service.rb @@ -15,7 +15,7 @@ module Packages tags_to_create = @tags - existing_tags @package.tags.with_name(tags_to_destroy).delete_all if tags_to_destroy.any? - ::Gitlab::Database.main.bulk_insert(Packages::Tag.table_name, rows(tags_to_create)) if tags_to_create.any? # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert(Packages::Tag.table_name, rows(tags_to_create)) if tags_to_create.any? # rubocop:disable Gitlab/BulkInsert end private diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb index a5ee7173bdf..e5d40b60747 100644 --- a/app/services/projects/alerting/notify_service.rb +++ b/app/services/projects/alerting/notify_service.rb @@ -5,6 +5,7 @@ module Projects class NotifyService extend ::Gitlab::Utils::Override include ::AlertManagement::AlertProcessing + include ::AlertManagement::Responses def initialize(project, payload) @project = project @@ -23,7 +24,7 @@ module Projects complete_post_processing_tasks - ServiceResponse.success + success(alert) end private @@ -46,18 +47,6 @@ module Projects def valid_token?(token) token == integration.token end - - def bad_request - ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) - end - - def unauthorized - ServiceResponse.error(message: 'Unauthorized', http_status: :unauthorized) - end - - def forbidden - ServiceResponse.error(message: 'Forbidden', http_status: :forbidden) - end end end end diff --git a/app/services/projects/all_issues_count_service.rb b/app/services/projects/all_issues_count_service.rb new file mode 100644 index 00000000000..15352b14d3e --- /dev/null +++ b/app/services/projects/all_issues_count_service.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Projects + # Service class for counting and caching the number of all issues of a + # project. + class AllIssuesCountService < Projects::CountService + def relation_for_count + @project.issues + end + + def cache_key_name + 'all_issues_count' + end + end +end diff --git a/app/services/projects/all_merge_requests_count_service.rb b/app/services/projects/all_merge_requests_count_service.rb new file mode 100644 index 00000000000..db0bab3f799 --- /dev/null +++ b/app/services/projects/all_merge_requests_count_service.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Projects + # Service class for counting and caching the number of all merge requests of + # a project. + class AllMergeRequestsCountService < Projects::CountService + def relation_for_count + @project.merge_requests + end + + def cache_key_name + 'all_merge_requests_count' + end + end +end diff --git a/app/services/projects/container_repository/cache_tags_created_at_service.rb b/app/services/projects/container_repository/cache_tags_created_at_service.rb deleted file mode 100644 index 3a5346d7a23..00000000000 --- a/app/services/projects/container_repository/cache_tags_created_at_service.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Projects - module ContainerRepository - class CacheTagsCreatedAtService - def initialize(container_repository) - @container_repository = container_repository - @cached_tag_names = Set.new - end - - def populate(tags) - return if tags.empty? - - # This will load all tags in one Redis roundtrip - # the maximum number of tags is configurable and is set to 200 by default. - # https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/packages/container_registry/index.md#set-cleanup-limits-to-conserve-resources - keys = tags.map(&method(:cache_key)) - cached_tags_count = 0 - - ::Gitlab::Redis::Cache.with do |redis| - tags.zip(redis.mget(keys)).each do |tag, created_at| - next unless created_at - - tag.created_at = DateTime.rfc3339(created_at) - @cached_tag_names << tag.name - cached_tags_count += 1 - end - end - - cached_tags_count - end - - def insert(tags, max_ttl_in_seconds) - return unless max_ttl_in_seconds - return if tags.empty? - - # tags with nil created_at are not cacheable - # tags already cached don't need to be cached again - cacheable_tags = tags.select do |tag| - tag.created_at.present? && !tag.name.in?(@cached_tag_names) - end - - return if cacheable_tags.empty? - - now = Time.zone.now - - ::Gitlab::Redis::Cache.with do |redis| - # we use a pipeline instead of a MSET because each tag has - # a specific ttl - redis.pipelined do - cacheable_tags.each do |tag| - created_at = tag.created_at - # ttl is the max_ttl_in_seconds reduced by the number - # of seconds that the tag has already existed - ttl = max_ttl_in_seconds - (now - created_at).seconds - ttl = ttl.to_i - redis.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0 - end - end - end - end - - private - - def cache_key(tag) - "container_repository:{#{@container_repository.id}}:tag:#{tag.name}:created_at" - end - end - end -end diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index 3a60de0f1ee..1a788abac12 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -140,14 +140,13 @@ module Projects def cache strong_memoize(:cache) do - ::Projects::ContainerRepository::CacheTagsCreatedAtService.new(@container_repository) + ::Gitlab::ContainerRepository::Tags::Cache.new(@container_repository) end end def caching_enabled? container_expiration_policy && - older_than.present? && - Feature.enabled?(:container_registry_expiration_policies_caching, @project) + older_than.present? end def throttling_enabled? diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 1536f0a22b8..1d187b140ef 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -45,7 +45,7 @@ module Projects if namespace_id # Find matching namespace and check if it allowed # for current user if namespace_id passed. - unless current_user.can?(:create_projects, project_namespace) + unless current_user.can?(:create_projects, parent_namespace) @project.namespace_id = nil deny_namespace return @project @@ -136,7 +136,7 @@ module Projects access_level: group_access_level) end - AuthorizedProjectUpdate::ProjectCreateWorker.perform_async(@project.id) + AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(@project.id) # AuthorizedProjectsWorker uses an exclusive lease per user but # specialized workers might have synchronization issues. Until we # compare the inconsistency rates of both approaches, we still run @@ -227,14 +227,14 @@ module Projects def extra_attributes_for_measurement { current_user: current_user&.name, - project_full_path: "#{project_namespace&.full_path}/#{@params[:path]}" + project_full_path: "#{parent_namespace&.full_path}/#{@params[:path]}" } end private - def project_namespace - @project_namespace ||= Namespace.find_by_id(@params[:namespace_id]) || current_user.namespace + def parent_namespace + @parent_namespace ||= Namespace.find_by_id(@params[:namespace_id]) || current_user.namespace end def create_from_template? diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 27f813f4661..b7ed9202b01 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -152,9 +152,12 @@ module Projects deleted_count = project.commit_statuses.delete_all - if deleted_count > 0 - Gitlab::AppLogger.info "Projects::DestroyService - Project #{project.id} - #{deleted_count} leftover commit statuses" - end + Gitlab::AppLogger.info( + class: 'Projects::DestroyService', + project_id: project.id, + message: 'leftover commit statuses', + orphaned_commit_status_count: deleted_count + ) end # The project can have multiple webhooks with hundreds of thousands of web_hook_logs. diff --git a/app/services/projects/detect_repository_languages_service.rb b/app/services/projects/detect_repository_languages_service.rb index 0356a6b0ccd..9db0b71d106 100644 --- a/app/services/projects/detect_repository_languages_service.rb +++ b/app/services/projects/detect_repository_languages_service.rb @@ -21,7 +21,7 @@ module Projects .update_all(share: update[:share]) end - Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert RepositoryLanguage.table_name, detection.insertions(matching_programming_languages) ) diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 64c0f1ff4ac..b1a2182fbdc 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -36,6 +36,7 @@ module Projects private attr_accessor :shared + attr_reader :logger def execute_after_export_action(after_export_strategy) return unless after_export_strategy @@ -74,7 +75,11 @@ module Projects end def project_tree_saver - tree_saver_class.new(project: project, current_user: current_user, shared: shared, params: params) + tree_saver_class.new(project: project, + current_user: current_user, + shared: shared, + params: params, + logger: logger) end def tree_saver_class @@ -116,7 +121,7 @@ module Projects end def notify_success - @logger.info( + logger.info( message: 'Project successfully exported', project_name: project.name, project_id: project.id @@ -124,7 +129,7 @@ module Projects end def notify_error - @logger.error( + logger.error( message: 'Project export error', export_errors: shared.errors.join(', '), project_name: project.name, diff --git a/app/services/projects/lfs_pointers/lfs_link_service.rb b/app/services/projects/lfs_pointers/lfs_link_service.rb index 7c00b9e6105..cf3cc5cd8e0 100644 --- a/app/services/projects/lfs_pointers/lfs_link_service.rb +++ b/app/services/projects/lfs_pointers/lfs_link_service.rb @@ -38,7 +38,7 @@ module Projects rows = existent_lfs_objects .not_linked_to_project(project) .map { |existing_lfs_object| { project_id: project.id, lfs_object_id: existing_lfs_object.id } } - Gitlab::Database.main.bulk_insert(:lfs_objects_projects, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(:lfs_objects_projects, rows) # rubocop:disable Gitlab/BulkInsert iterations += 1 linked_existing_objects += existent_lfs_objects.map(&:oid) diff --git a/app/services/projects/participants_service.rb b/app/services/projects/participants_service.rb index 1616a8a4062..152590fffff 100644 --- a/app/services/projects/participants_service.rb +++ b/app/services/projects/participants_service.rb @@ -36,17 +36,9 @@ module Projects private def project_members_through_invited_groups - groups_with_ancestors = if ::Feature.enabled?(:linear_participants_service_ancestor_scopes, current_user, default_enabled: :yaml) - visible_groups.self_and_ancestors - else - Gitlab::ObjectHierarchy - .new(visible_groups) - .base_and_ancestors - end - GroupMember .active_without_invites_and_requests - .with_source_id(groups_with_ancestors.pluck_primary_key) + .with_source_id(visible_groups.self_and_ancestors.pluck_primary_key) end def visible_groups diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index c1bf2e68436..56f65718d24 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -6,6 +6,7 @@ module Projects class NotifyService include Gitlab::Utils::StrongMemoize include ::IncidentManagement::Settings + include ::AlertManagement::Responses # This set of keys identifies a payload as a valid Prometheus # payload and thus processable by this service. See also @@ -27,9 +28,9 @@ module Projects return unprocessable_entity unless self.class.processable?(payload) return unauthorized unless valid_alert_manager_token?(token, integration) - process_prometheus_alerts + alert_responses = process_prometheus_alerts - ServiceResponse.success + alert_response(alert_responses) end def self.processable?(payload) @@ -128,23 +129,17 @@ module Projects end def process_prometheus_alerts - alerts.each do |alert| + alerts.map do |alert| AlertManagement::ProcessPrometheusAlertService .new(project, alert.to_h) .execute end end - def bad_request - ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) - end - - def unauthorized - ServiceResponse.error(message: 'Unauthorized', http_status: :unauthorized) - end + def alert_response(alert_responses) + alerts = alert_responses.map { |resp| resp.payload[:alert] }.compact - def unprocessable_entity - ServiceResponse.error(message: 'Unprocessable Entity', http_status: :unprocessable_entity) + success(alerts) end end end diff --git a/app/services/resource_events/base_synthetic_notes_builder_service.rb b/app/services/resource_events/base_synthetic_notes_builder_service.rb index 5939b9d2f9c..192d40129a3 100644 --- a/app/services/resource_events/base_synthetic_notes_builder_service.rb +++ b/app/services/resource_events/base_synthetic_notes_builder_service.rb @@ -24,10 +24,18 @@ module ResourceEvents private def apply_common_filters(events) + events = apply_pagination(events) events = apply_last_fetched_at(events) apply_fetch_until(events) end + def apply_pagination(events) + return events if params[:paginated_notes].nil? + return events.none if params[:paginated_notes][table_name].blank? + + events.id_in(params[:paginated_notes][table_name].map(&:id)) + end + def apply_last_fetched_at(events) return events unless params[:last_fetched_at].present? @@ -47,5 +55,9 @@ module ResourceEvents resource.project || resource.group end end + + def table_name + raise NotImplementedError + end end end diff --git a/app/services/resource_events/change_labels_service.rb b/app/services/resource_events/change_labels_service.rb index bc2d3a946cc..03ac839c509 100644 --- a/app/services/resource_events/change_labels_service.rb +++ b/app/services/resource_events/change_labels_service.rb @@ -23,7 +23,7 @@ module ResourceEvents label_hash.merge(label_id: label.id, action: ResourceLabelEvent.actions['remove']) end - Gitlab::Database.main.bulk_insert(ResourceLabelEvent.table_name, labels) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(ResourceLabelEvent.table_name, labels) # rubocop:disable Gitlab/BulkInsert resource.expire_note_etag_cache Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_label_changed_action(author: user) if resource.is_a?(Issue) diff --git a/app/services/resource_events/synthetic_label_notes_builder_service.rb b/app/services/resource_events/synthetic_label_notes_builder_service.rb index 5915ea938cf..0e5d945d13c 100644 --- a/app/services/resource_events/synthetic_label_notes_builder_service.rb +++ b/app/services/resource_events/synthetic_label_notes_builder_service.rb @@ -23,5 +23,9 @@ module ResourceEvents events.group_by { |event| event.discussion_id } end + + def table_name + 'resource_label_events' + end end end diff --git a/app/services/resource_events/synthetic_milestone_notes_builder_service.rb b/app/services/resource_events/synthetic_milestone_notes_builder_service.rb index 10acf94e22b..0e2b171e192 100644 --- a/app/services/resource_events/synthetic_milestone_notes_builder_service.rb +++ b/app/services/resource_events/synthetic_milestone_notes_builder_service.rb @@ -21,5 +21,9 @@ module ResourceEvents events = resource.resource_milestone_events.includes(user: :status) # rubocop: disable CodeReuse/ActiveRecord apply_common_filters(events) end + + def table_name + 'resource_milestone_events' + end end end diff --git a/app/services/resource_events/synthetic_state_notes_builder_service.rb b/app/services/resource_events/synthetic_state_notes_builder_service.rb index 71d40200365..e17882b00de 100644 --- a/app/services/resource_events/synthetic_state_notes_builder_service.rb +++ b/app/services/resource_events/synthetic_state_notes_builder_service.rb @@ -16,5 +16,9 @@ module ResourceEvents events = resource.resource_state_events.includes(user: :status) # rubocop: disable CodeReuse/ActiveRecord apply_common_filters(events) end + + def table_name + 'resource_state_events' + end end end diff --git a/app/services/search_service.rb b/app/services/search_service.rb index cce7821a226..4ba1b3ade86 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -75,6 +75,10 @@ class SearchService search_results.highlight_map(scope) end + def search_aggregations + search_results.aggregations(scope) + end + private def page diff --git a/app/services/security/ci_configuration/sast_iac_create_service.rb b/app/services/security/ci_configuration/sast_iac_create_service.rb new file mode 100644 index 00000000000..80e9cf963da --- /dev/null +++ b/app/services/security/ci_configuration/sast_iac_create_service.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Security + module CiConfiguration + class SastIacCreateService < ::Security::CiConfiguration::BaseCreateService + private + + def action + Security::CiConfiguration::SastIacBuildAction.new(project.auto_devops_enabled?, existing_gitlab_ci_content).generate + end + + def next_branch + 'set-sast-iac-config' + end + + def message + _('Configure SAST IaC in `.gitlab-ci.yml`, creating this file if it does not already exist') + end + + def description + _('Configure SAST IaC in `.gitlab-ci.yml` using the GitLab managed template. You can [add variable overrides](https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings) to customize SAST IaC settings.') + end + end + end +end diff --git a/app/services/snippets/update_service.rb b/app/services/snippets/update_service.rb index d83b21271c0..76d5063c337 100644 --- a/app/services/snippets/update_service.rb +++ b/app/services/snippets/update_service.rb @@ -148,3 +148,5 @@ module Snippets end end end + +Snippets::UpdateService.prepend_mod_with('Snippets::UpdateService') diff --git a/app/services/spam/spam_verdict_service.rb b/app/services/spam/spam_verdict_service.rb index 8d995631db6..c8bdcf4310b 100644 --- a/app/services/spam/spam_verdict_service.rb +++ b/app/services/spam/spam_verdict_service.rb @@ -73,18 +73,12 @@ module Spam begin result, attribs, _error = spamcheck_client.issue_spam?(spam_issue: target, user: user, context: context) - return [nil, attribs] unless result - # @TODO log if error is not nil https://gitlab.com/gitlab-org/gitlab/-/issues/329545 - return [result, attribs] if result == NOOP || attribs["monitorMode"] == "true" + return [nil, attribs] unless result + + [result, attribs] - # Duplicate logic with Akismet logic in #akismet_verdict - if Gitlab::Recaptcha.enabled? && result != ALLOW - [CONDITIONAL_ALLOW, attribs] - else - [result, attribs] - end rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) diff --git a/app/services/suggestions/create_service.rb b/app/services/suggestions/create_service.rb index eb98ed57d55..239cd86e0ec 100644 --- a/app/services/suggestions/create_service.rb +++ b/app/services/suggestions/create_service.rb @@ -25,7 +25,7 @@ module Suggestions end rows.in_groups_of(100, false) do |rows| - Gitlab::Database.main.bulk_insert('suggestions', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('suggestions', rows) # rubocop:disable Gitlab/BulkInsert end Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter.track_add_suggestion_action(note: @note) diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index e5080718b69..dc5cf0fe554 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -327,6 +327,10 @@ module SystemNoteService ::SystemNotes::IncidentService.new(noteable: incident, project: incident.project, author: author).change_incident_severity end + def resolve_incident_status(incident, author) + ::SystemNotes::IncidentService.new(noteable: incident, project: incident.project, author: author).resolve_incident_status + end + def log_resolving_alert(alert, monitoring_tool) ::SystemNotes::AlertManagementService.new(noteable: alert, project: alert.project).log_resolving_alert(monitoring_tool) end diff --git a/app/services/system_notes/incident_service.rb b/app/services/system_notes/incident_service.rb index 4628662f0e9..785291e0637 100644 --- a/app/services/system_notes/incident_service.rb +++ b/app/services/system_notes/incident_service.rb @@ -25,5 +25,11 @@ module SystemNotes ) end end + + def resolve_incident_status + body = 'changed the status to **Resolved** by closing the incident' + + create_note(NoteSummary.new(noteable, project, author, body, action: 'status')) + end end end diff --git a/app/services/system_notes/issuables_service.rb b/app/services/system_notes/issuables_service.rb index 62aead352aa..94629ae7609 100644 --- a/app/services/system_notes/issuables_service.rb +++ b/app/services/system_notes/issuables_service.rb @@ -176,7 +176,13 @@ module SystemNotes body = cross_reference_note_content(gfm_reference) if noteable.is_a?(ExternalIssue) - noteable.project.external_issue_tracker.create_cross_reference_note(noteable, mentioner, author) + Integrations::CreateExternalCrossReferenceWorker.perform_async( + noteable.project_id, + noteable.id, + mentioner.class.name, + mentioner.id, + author.id + ) else track_cross_reference_action create_note(NoteSummary.new(noteable, noteable.project, author, body, action: 'cross_reference')) diff --git a/app/services/tasks_to_be_done/base_service.rb b/app/services/tasks_to_be_done/base_service.rb new file mode 100644 index 00000000000..a5648ad10c4 --- /dev/null +++ b/app/services/tasks_to_be_done/base_service.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module TasksToBeDone + class BaseService < ::IssuableBaseService + LABEL_PREFIX = 'tasks to be done' + + def initialize(project:, current_user:, assignee_ids: []) + params = { + assignee_ids: assignee_ids, + title: title, + description: description, + add_labels: label_name + } + super(project: project, current_user: current_user, params: params) + end + + def execute + if (issue = existing_task_issue) + update_service = Issues::UpdateService.new(project: project, current_user: current_user, params: { add_assignee_ids: params[:assignee_ids] }) + update_service.execute(issue) + else + build_service = Issues::BuildService.new(project: project, current_user: current_user, params: params) + create(build_service.execute) + end + end + + private + + def existing_task_issue + IssuesFinder.new( + current_user, + project_id: project.id, + state: 'opened', + non_archived: true, + label_name: label_name + ).execute.last + end + + def title + raise NotImplementedError + end + + def description + raise NotImplementedError + end + + def label_suffix + raise NotImplementedError + end + + def label_name + "#{LABEL_PREFIX}:#{label_suffix}" + end + end +end diff --git a/app/services/tasks_to_be_done/create_ci_task_service.rb b/app/services/tasks_to_be_done/create_ci_task_service.rb new file mode 100644 index 00000000000..025ca2feb8e --- /dev/null +++ b/app/services/tasks_to_be_done/create_ci_task_service.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module TasksToBeDone + class CreateCiTaskService < BaseService + protected + + def title + 'Set up CI/CD' + end + + def description + <<~DESCRIPTION + GitLab CI/CD is a tool built into GitLab for software development through the [continuous methodologies](https://docs.gitlab.com/ee/ci/introduction/index.html#introduction-to-cicd-methodologies): + + * Continuous Integration (CI) + * Continuous Delivery (CD) + * Continuous Deployment (CD) + + Continuous Integration works by pushing small changes to your application’s codebase hosted in a Git repository, and, to every push, run a pipeline of scripts to build, test, and validate the code changes before merging them into the main branch. + + Continuous Delivery and Deployment consist of a step further CI, deploying your application to production at every push to the default branch of the repository. + + These methodologies allow you to catch bugs and errors early in the development cycle, ensuring that all the code deployed to production complies with the code standards you established for your app. + + * :book: [Read the documentation](https://docs.gitlab.com/ee/ci/introduction/index.html) + * :clapper: [Watch a Demo](https://www.youtube.com/watch?v=1iXFbchozdY) + + ## Next steps + + * [ ] To start we recommend reviewing the following documentation: + * [ ] [How GitLab CI/CD works.](https://docs.gitlab.com/ee/ci/introduction/index.html#how-gitlab-cicd-works) + * [ ] [Fundamental pipeline architectures.](https://docs.gitlab.com/ee/ci/pipelines/pipeline_architectures.html) + * [ ] [GitLab CI/CD basic workflow.](https://docs.gitlab.com/ee/ci/introduction/index.html#basic-cicd-workflow) + * [ ] [Step-by-step guide for writing .gitlab-ci.yml for the first time.](https://docs.gitlab.com/ee/user/project/pages/getting_started_part_four.html) + * [ ] When you're ready select **Projects** (in the top navigation bar) > **Your projects** > select the Project you've already created. + * [ ] Select **CI / CD** in the left navigation to start setting up CI / CD in your project. + DESCRIPTION + end + + def label_suffix + 'ci' + end + end +end diff --git a/app/services/tasks_to_be_done/create_code_task_service.rb b/app/services/tasks_to_be_done/create_code_task_service.rb new file mode 100644 index 00000000000..dc3b9366a66 --- /dev/null +++ b/app/services/tasks_to_be_done/create_code_task_service.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module TasksToBeDone + class CreateCodeTaskService < BaseService + protected + + def title + 'Create or import your code into your Project (Repository)' + end + + def description + <<~DESCRIPTION + You've already created your Group and Project within GitLab; we'll quickly review this hierarchy below. Once you're within your project you can easily create or import repositories. + + **With GitLab Groups, you can:** + + * Create one or multiple Projects for hosting your codebase (repositories). + * Assemble related projects together. + * Grant members access to several projects at once. + + Groups can also be nested in subgroups. + + Read more about groups in our [documentation](https://docs.gitlab.com/ee/user/group/). + + **Within GitLab Projects, you can** + + * Use it as an issue tracker. + * Collaborate on code. + * Continuously build, test, and deploy your app with built-in GitLab CI/CD. + + You can also import an existing repository by providing the Git URL. + + * :book: [Read the documentation](https://docs.gitlab.com/ee/user/project/index.html). + + ## Next steps + + Create or import your first repository into the project you created: + + * [ ] Click **Projects** in the top navigation bar, then click **Your projects**. + * [ ] Select the Project that you created, then select **Repository**. + * [ ] Once on the Repository page you can select the **+** icon to add or import files. + * [ ] You can review our full documentation on creating [repositories](https://docs.gitlab.com/ee/user/project/repository/) in GitLab. + + :tada: All done, you can close this issue! + DESCRIPTION + end + + def label_suffix + 'code' + end + end +end diff --git a/app/services/tasks_to_be_done/create_issues_task_service.rb b/app/services/tasks_to_be_done/create_issues_task_service.rb new file mode 100644 index 00000000000..a2de6852868 --- /dev/null +++ b/app/services/tasks_to_be_done/create_issues_task_service.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module TasksToBeDone + class CreateIssuesTaskService < BaseService + protected + + def title + 'Create/import issues (tickets) to collaborate on ideas and plan work' + end + + def description + <<~DESCRIPTION + Issues allow you and your team to discuss proposals before, and during, their implementation. They can be used for a variety of other purposes, customized to your needs and workflow. + + Issues are always associated with a specific project. If you have multiple projects in a group, you can view all the issues at the group level. [You can review our full Issue documentation here.](https://docs.gitlab.com/ee/user/project/issues/) + + If you have existing issues or equivalent tickets you can import them as long as they are formatted as a CSV file, [the import process is covered here](https://docs.gitlab.com/ee/user/project/issues/csv_import.html). + + **Common use cases include:** + + * Discussing the implementation of a new idea + * Tracking tasks and work status + * Accepting feature proposals, questions, support requests, or bug reports + * Elaborating on new code implementations + + ## Next steps + + * [ ] Select **Projects** in the top navigation > **Your Projects** > select the Project you've already created. + * [ ] Once you've selected that project, you can select **Issues** in the left navigation, then click **New issue**. + * [ ] Fill in the title and description in the **New issue** page. + * [ ] Click on **Create issue**. + + Pro tip: When you're in a group or project you can always utilize the **+** icon in the top navigation (located to the left of the search bar) to quickly create new issues. + + That's it! You can close this issue. + DESCRIPTION + end + + def label_suffix + 'issues' + end + end +end diff --git a/app/services/todo_service.rb b/app/services/todo_service.rb index 71bb813f384..091f441831a 100644 --- a/app/services/todo_service.rb +++ b/app/services/todo_service.rb @@ -217,6 +217,11 @@ class TodoService create_todos(reviewers, attributes) end + def create_attention_requested_todo(target, author, users) + attributes = attributes_for_todo(target.project, target, author, Todo::ATTENTION_REQUESTED) + create_todos(users, attributes) + end + private def create_todos(users, attributes) diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb index 4ec875098fa..1634cc017ae 100644 --- a/app/services/users/destroy_service.rb +++ b/app/services/users/destroy_service.rb @@ -65,7 +65,10 @@ module Users user.destroy_dependent_associations_in_batches(exclude: [:snippets]) # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing - user_data = user.destroy + user_data = nil + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/340260') do + user_data = user.destroy + end namespace.destroy user_data diff --git a/app/services/users/refresh_authorized_projects_service.rb b/app/services/users/refresh_authorized_projects_service.rb index 1850fa9747d..2d9766c3c56 100644 --- a/app/services/users/refresh_authorized_projects_service.rb +++ b/app/services/users/refresh_authorized_projects_service.rb @@ -67,10 +67,8 @@ module Users def update_authorizations(remove = [], add = []) log_refresh_details(remove, add) - User.transaction do - user.remove_project_authorizations(remove) unless remove.empty? - ProjectAuthorization.insert_authorizations(add) unless add.empty? - end + user.remove_project_authorizations(remove) unless remove.empty? + ProjectAuthorization.insert_authorizations(add) unless add.empty? # Since we batch insert authorization rows, Rails' associations may get # out of sync. As such we force a reload of the User object. diff --git a/app/services/users/upsert_credit_card_validation_service.rb b/app/services/users/upsert_credit_card_validation_service.rb index 86b5b923418..61cf598f178 100644 --- a/app/services/users/upsert_credit_card_validation_service.rb +++ b/app/services/users/upsert_credit_card_validation_service.rb @@ -12,6 +12,7 @@ module Users credit_card_validated_at: params.fetch(:credit_card_validated_at), expiration_date: get_expiration_date(params), last_digits: Integer(params.fetch(:credit_card_mask_number), 10), + network: params.fetch(:credit_card_type), holder_name: params.fetch(:credit_card_holder_name) } |