diff options
Diffstat (limited to 'app/services')
149 files changed, 3166 insertions, 857 deletions
diff --git a/app/services/access_token_validation_service.rb b/app/services/access_token_validation_service.rb index 851d862c0cf..eb2e66a9285 100644 --- a/app/services/access_token_validation_service.rb +++ b/app/services/access_token_validation_service.rb @@ -17,21 +17,21 @@ class AccessTokenValidationService def validate(scopes: []) if token.expired? - return EXPIRED + EXPIRED elsif token.revoked? - return REVOKED + REVOKED elsif !self.include_any_scope?(scopes) - return INSUFFICIENT_SCOPE + INSUFFICIENT_SCOPE elsif token.respond_to?(:impersonation) && token.impersonation && !Gitlab.config.gitlab.impersonation_enabled - return IMPERSONATION_DISABLED + IMPERSONATION_DISABLED else - return VALID + VALID end end diff --git a/app/services/admin/propagate_integration_service.rb b/app/services/admin/propagate_integration_service.rb index 084b103ee3b..e21bb03ed68 100644 --- a/app/services/admin/propagate_integration_service.rb +++ b/app/services/admin/propagate_integration_service.rb @@ -64,7 +64,7 @@ module Admin def create_integration_for_projects_without_integration loop do - batch = Project.uncached { project_ids_without_integration } + batch = Project.uncached { Project.ids_without_integration(integration, BATCH_SIZE) } bulk_create_from_integration(batch) unless batch.empty? @@ -114,22 +114,6 @@ module Admin integration.type == 'ExternalWikiService' end - # rubocop: disable CodeReuse/ActiveRecord - def project_ids_without_integration - services = Service - .select('1') - .where('services.project_id = projects.id') - .where(type: integration.type) - - Project - .where('NOT EXISTS (?)', services) - .where(pending_delete: false) - .where(archived: false) - .limit(BATCH_SIZE) - .pluck(:id) - end - # rubocop: enable CodeReuse/ActiveRecord - def service_hash @service_hash ||= integration.to_service_hash .tap { |json| json['inherit_from_id'] = integration.id } diff --git a/app/services/alert_management/alerts/todo/create_service.rb b/app/services/alert_management/alerts/todo/create_service.rb new file mode 100644 index 00000000000..87af943fdc2 --- /dev/null +++ b/app/services/alert_management/alerts/todo/create_service.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module AlertManagement + module Alerts + module Todo + class CreateService + # @param alert [AlertManagement::Alert] + # @param current_user [User] + def initialize(alert, current_user) + @alert = alert + @current_user = current_user + end + + def execute + return error_no_permissions unless allowed? + + todos = TodoService.new.mark_todo(alert, current_user) + todo = todos&.first + + return error_existing_todo unless todo + + success(todo) + end + + private + + attr_reader :alert, :current_user + + def allowed? + current_user&.can?(:update_alert_management_alert, alert) + end + + def error(message) + ServiceResponse.error(payload: { alert: alert, todo: nil }, message: message) + end + + def success(todo) + ServiceResponse.success(payload: { alert: alert, todo: todo }) + end + + def error_no_permissions + error(_('You have insufficient permissions to create a Todo for this alert')) + end + + def error_existing_todo + error(_('You already have pending todo for this alert')) + end + end + end + end +end diff --git a/app/services/alert_management/alerts/update_service.rb b/app/services/alert_management/alerts/update_service.rb index ffabbb37289..0b7216cd9f8 100644 --- a/app/services/alert_management/alerts/update_service.rb +++ b/app/services/alert_management/alerts/update_service.rb @@ -12,17 +12,20 @@ module AlertManagement @alert = alert @current_user = current_user @params = params + @param_errors = [] end def execute return error_no_permissions unless allowed? - return error_no_updates if params.empty? - filter_assignees + filter_params + return error_invalid_params if param_errors.any? + + # Save old assignees for system notes old_assignees = alert.assignees.to_a if alert.update(params) - process_assignement(old_assignees) + handle_changes(old_assignees: old_assignees) success else @@ -32,16 +35,13 @@ module AlertManagement private - attr_reader :alert, :current_user, :params + attr_reader :alert, :current_user, :params, :param_errors + delegate :resolved?, to: :alert def allowed? current_user&.can?(:update_alert_management_alert, alert) end - def assignee_todo_allowed? - assignee&.can?(:read_alert_management_alert, alert) - end - def todo_service strong_memoize(:todo_service) do TodoService.new @@ -60,39 +60,122 @@ module AlertManagement error(_('You have no permissions')) end - def error_no_updates - error(_('Please provide attributes to update')) + def error_invalid_params + error(param_errors.to_sentence) + end + + def add_param_error(message) + param_errors << message + end + + def filter_params + param_errors << _('Please provide attributes to update') if params.empty? + + filter_status + filter_assignees + filter_duplicate + end + + def handle_changes(old_assignees:) + handle_assignement(old_assignees) if params[:assignees] + handle_status_change if params[:status_event] end # ----- Assignee-related behavior ------ def filter_assignees return if params[:assignees].nil? - params[:assignees] = Array(assignee) + # Always take first assignee while multiple are not currently supported + params[:assignees] = Array(params[:assignees].first) + + param_errors << _('Assignee has no permissions') if unauthorized_assignees? end - def assignee - strong_memoize(:assignee) do - # Take first assignee while multiple are not currently supported - params[:assignees]&.first - end + def unauthorized_assignees? + params[:assignees]&.any? { |user| !user.can?(:read_alert_management_alert, alert) } end - def process_assignement(old_assignees) + def handle_assignement(old_assignees) assign_todo add_assignee_system_note(old_assignees) end def assign_todo - # Remove check in follow-up issue https://gitlab.com/gitlab-org/gitlab/-/issues/222672 - return unless assignee_todo_allowed? - todo_service.assign_alert(alert, current_user) end def add_assignee_system_note(old_assignees) SystemNoteService.change_issuable_assignees(alert, alert.project, current_user, old_assignees) end + + # ------ Status-related behavior ------- + def filter_status + return unless params[:status] + + status_event = AlertManagement::Alert::STATUS_EVENTS[status_key] + + unless status_event + param_errors << _('Invalid status') + return + end + + params[:status_event] = status_event + end + + def status_key + strong_memoize(:status_key) do + status = params.delete(:status) + AlertManagement::Alert::STATUSES.key(status) + end + end + + def handle_status_change + add_status_change_system_note + resolve_todos if resolved? + end + + def add_status_change_system_note + SystemNoteService.change_alert_status(alert, current_user) + end + + def resolve_todos + todo_service.resolve_todos_for_target(alert, current_user) + end + + def filter_duplicate + # Only need to check if changing to an open status + return unless params[:status_event] && AlertManagement::Alert::OPEN_STATUSES.include?(status_key) + + param_errors << unresolved_alert_error if duplicate_alert? + end + + def duplicate_alert? + return if alert.fingerprint.blank? + + open_alerts.any? && open_alerts.exclude?(alert) + end + + def open_alerts + strong_memoize(:open_alerts) do + AlertManagement::Alert.for_fingerprint(alert.project, alert.fingerprint).open + end + end + + def unresolved_alert_error + _('An %{link_start}alert%{link_end} with the same fingerprint is already open. ' \ + 'To change the status of this alert, resolve the linked alert.' + ) % open_alert_url_params + end + + def open_alert_url_params + open_alert = open_alerts.first + alert_path = Gitlab::Routing.url_helpers.details_project_alert_management_path(alert.project, open_alert) + + { + link_start: '<a href="%{url}">'.html_safe % { url: alert_path }, + link_end: '</a>'.html_safe + } + end end end end diff --git a/app/services/alert_management/create_alert_issue_service.rb b/app/services/alert_management/create_alert_issue_service.rb index beacd240b08..6ea3fd867ef 100644 --- a/app/services/alert_management/create_alert_issue_service.rb +++ b/app/services/alert_management/create_alert_issue_service.rb @@ -2,6 +2,8 @@ module AlertManagement class CreateAlertIssueService + include Gitlab::Utils::StrongMemoize + # @param alert [AlertManagement::Alert] # @param user [User] def initialize(alert, user) @@ -13,18 +15,20 @@ module AlertManagement return error_no_permissions unless allowed? return error_issue_already_exists if alert.issue - result = create_issue(alert, user, alert_payload) - @issue = result[:issue] + result = create_issue + issue = result.payload[:issue] + + return error(result.message, issue) if result.error? + return error(object_errors(alert), issue) unless associate_alert_with_issue(issue) - return error(result[:message]) if result[:status] == :error - return error(alert.errors.full_messages.to_sentence) unless update_alert_issue_id + SystemNoteService.new_alert_issue(alert, issue, user) - success + result end private - attr_reader :alert, :user, :issue + attr_reader :alert, :user delegate :project, to: :alert @@ -32,29 +36,36 @@ module AlertManagement user.can?(:create_issue, project) end - def create_issue(alert, user, alert_payload) - ::IncidentManagement::CreateIssueService - .new(project, alert_payload, user) - .execute(skip_settings_check: true) - end + def create_issue + label_result = find_or_create_incident_label - def alert_payload - if alert.prometheus? - alert.payload - else - Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h) - end + # Create an unlabelled issue if we couldn't create the label + # due to a race condition. + # See https://gitlab.com/gitlab-org/gitlab-foss/issues/65042 + extra_params = label_result.success? ? { label_ids: [label_result.payload[:label].id] } : {} + + issue = Issues::CreateService.new( + project, + user, + title: alert_presenter.title, + description: alert_presenter.issue_description, + **extra_params + ).execute + + return error(object_errors(issue), issue) unless issue.valid? + + success(issue) end - def update_alert_issue_id + def associate_alert_with_issue(issue) alert.update(issue_id: issue.id) end - def success + def success(issue) ServiceResponse.success(payload: { issue: issue }) end - def error(message) + def error(message, issue = nil) ServiceResponse.error(payload: { issue: issue }, message: message) end @@ -65,5 +76,19 @@ module AlertManagement def error_no_permissions error(_('You have no permissions')) end + + def alert_presenter + strong_memoize(:alert_presenter) do + alert.present + end + end + + def find_or_create_incident_label + IncidentManagement::CreateIncidentLabelService.new(project, user).execute + end + + def object_errors(object) + object.errors.full_messages.to_sentence + end end end diff --git a/app/services/alert_management/process_prometheus_alert_service.rb b/app/services/alert_management/process_prometheus_alert_service.rb index 90fcbd95e4b..573d3914c05 100644 --- a/app/services/alert_management/process_prometheus_alert_service.rb +++ b/app/services/alert_management/process_prometheus_alert_service.rb @@ -66,7 +66,11 @@ module AlertManagement def process_resolved_alert_management_alert return if am_alert.blank? - return if am_alert.resolve(ends_at) + + if am_alert.resolve(ends_at) + close_issue(am_alert.issue) + return + end logger.warn( message: 'Unable to update AlertManagement::Alert status to resolved', @@ -75,12 +79,22 @@ module AlertManagement ) end + def close_issue(issue) + return if issue.blank? || issue.closed? + + Issues::CloseService + .new(project, User.alert_bot) + .execute(issue, system_note: false) + + SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed? + end + def logger @logger ||= Gitlab::AppLogger end def am_alert - @am_alert ||= AlertManagement::Alert.for_fingerprint(project, gitlab_fingerprint).first + @am_alert ||= AlertManagement::Alert.not_resolved.for_fingerprint(project, gitlab_fingerprint).first end def bad_request diff --git a/app/services/alert_management/update_alert_status_service.rb b/app/services/alert_management/update_alert_status_service.rb deleted file mode 100644 index a7ebddb82e0..00000000000 --- a/app/services/alert_management/update_alert_status_service.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module AlertManagement - class UpdateAlertStatusService - include Gitlab::Utils::StrongMemoize - - # @param alert [AlertManagement::Alert] - # @param user [User] - # @param status [Integer] Must match a value from AlertManagement::Alert::STATUSES - def initialize(alert, user, status) - @alert = alert - @user = user - @status = status - end - - def execute - return error_no_permissions unless allowed? - return error_invalid_status unless status_key - - if alert.update(status_event: status_event) - success - else - error(alert.errors.full_messages.to_sentence) - end - end - - private - - attr_reader :alert, :user, :status - - delegate :project, to: :alert - - def allowed? - user.can?(:update_alert_management_alert, project) - end - - def status_key - strong_memoize(:status_key) do - AlertManagement::Alert::STATUSES.key(status) - end - end - - def status_event - AlertManagement::Alert::STATUS_EVENTS[status_key] - end - - def success - ServiceResponse.success(payload: { alert: alert }) - end - - def error_no_permissions - error(_('You have no permissions')) - end - - def error_invalid_status - error(_('Invalid status')) - end - - def error(message) - ServiceResponse.error(payload: { alert: alert }, message: message) - end - end -end diff --git a/app/services/audit_event_service.rb b/app/services/audit_event_service.rb index fb309aed649..fef733a7d09 100644 --- a/app/services/audit_event_service.rb +++ b/app/services/audit_event_service.rb @@ -16,6 +16,7 @@ class AuditEventService @author = build_author(author) @entity = entity @details = details + @ip_address = (@details[:ip_address].presence || @author.current_sign_in_ip) end # Builds the @details attribute for authentication @@ -49,6 +50,8 @@ class AuditEventService private + attr_reader :ip_address + def build_author(author) case author when User @@ -61,6 +64,7 @@ class AuditEventService def base_payload { author_id: @author.id, + author_name: @author.name, entity_id: @entity.id, entity_type: @entity.class.name } diff --git a/app/services/authorized_project_update/project_create_service.rb b/app/services/authorized_project_update/project_create_service.rb index c17c0a033fe..5809315a066 100644 --- a/app/services/authorized_project_update/project_create_service.rb +++ b/app/services/authorized_project_update/project_create_service.rb @@ -21,7 +21,7 @@ module AuthorizedProjectUpdate { user_id: member.user_id, project_id: project.id, access_level: member.access_level } end - ProjectAuthorization.insert_all(attributes) + ProjectAuthorization.insert_all(attributes) unless attributes.empty? end ServiceResponse.success diff --git a/app/services/authorized_project_update/project_group_link_create_service.rb b/app/services/authorized_project_update/project_group_link_create_service.rb new file mode 100644 index 00000000000..db2db091374 --- /dev/null +++ b/app/services/authorized_project_update/project_group_link_create_service.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module AuthorizedProjectUpdate + class ProjectGroupLinkCreateService < BaseService + include Gitlab::Utils::StrongMemoize + + BATCH_SIZE = 1000 + + def initialize(project, group) + @project = project + @group = group + end + + def execute + group.members_from_self_and_ancestors_with_effective_access_level + .each_batch(of: BATCH_SIZE, column: :user_id) do |members| + existing_authorizations = existing_project_authorizations(members) + authorizations_to_create = [] + user_ids_to_delete = [] + + members.each do |member| + existing_access_level = existing_authorizations[member.user_id] + + if existing_access_level + # User might already have access to the project unrelated to the + # current project share + next if existing_access_level >= member.access_level + + user_ids_to_delete << member.user_id + end + + authorizations_to_create << { user_id: member.user_id, + project_id: project.id, + access_level: member.access_level } + end + + update_authorizations(user_ids_to_delete, authorizations_to_create) + end + + ServiceResponse.success + end + + private + + attr_reader :project, :group + + def existing_project_authorizations(members) + user_ids = members.map(&:user_id) + + ProjectAuthorization.where(project_id: project.id, user_id: user_ids) # rubocop: disable CodeReuse/ActiveRecord + .select(:user_id, :access_level) + .each_with_object({}) do |authorization, hash| + hash[authorization.user_id] = authorization.access_level + end + end + + def update_authorizations(user_ids_to_delete, authorizations_to_create) + ProjectAuthorization.transaction do + if user_ids_to_delete.any? + ProjectAuthorization.where(project_id: project.id, user_id: user_ids_to_delete) # rubocop: disable CodeReuse/ActiveRecord + .delete_all + end + + if authorizations_to_create.any? + ProjectAuthorization.insert_all(authorizations_to_create) + end + end + end + end +end diff --git a/app/services/auto_merge/base_service.rb b/app/services/auto_merge/base_service.rb index c4109765a1c..5c63dc34cb1 100644 --- a/app/services/auto_merge/base_service.rb +++ b/app/services/auto_merge/base_service.rb @@ -11,7 +11,7 @@ module AutoMerge yield if block_given? end - # Notify the event that auto merge is enabled or merge param is updated + notify(merge_request) AutoMergeProcessWorker.perform_async(merge_request.id) strategy.to_sym @@ -62,6 +62,10 @@ module AutoMerge private + # Overridden in child classes + def notify(merge_request) + end + def strategy strong_memoize(:strategy) do self.class.name.demodulize.remove('Service').underscore diff --git a/app/services/auto_merge/merge_when_pipeline_succeeds_service.rb b/app/services/auto_merge/merge_when_pipeline_succeeds_service.rb index 9ae5bd1b5ec..7e0298432ac 100644 --- a/app/services/auto_merge/merge_when_pipeline_succeeds_service.rb +++ b/app/services/auto_merge/merge_when_pipeline_succeeds_service.rb @@ -34,5 +34,13 @@ module AutoMerge merge_request.actual_head_pipeline&.active? end end + + private + + def notify(merge_request) + return unless Feature.enabled?(:mwps_notification, project) + + notification_service.async.merge_when_pipeline_succeeds(merge_request, current_user) if merge_request.saved_change_to_auto_merge_enabled? + end end end diff --git a/app/services/branches/delete_service.rb b/app/services/branches/delete_service.rb index ca2b4556b58..9bd5b343448 100644 --- a/app/services/branches/delete_service.rb +++ b/app/services/branches/delete_service.rb @@ -19,6 +19,7 @@ module Branches end if repository.rm_branch(current_user, branch_name) + unlock_artifacts(branch_name) ServiceResponse.success(message: 'Branch was deleted') else ServiceResponse.error( @@ -28,5 +29,11 @@ module Branches rescue Gitlab::Git::PreReceiveError => ex ServiceResponse.error(message: ex.message, http_status: 400) end + + private + + def unlock_artifacts(branch_name) + Ci::RefDeleteUnlockArtifactsWorker.perform_async(project.id, current_user.id, "#{::Gitlab::Git::BRANCH_REF_PREFIX}#{branch_name}") + end end end diff --git a/app/services/ci/authorize_job_artifact_service.rb b/app/services/ci/authorize_job_artifact_service.rb deleted file mode 100644 index 893e92d427c..00000000000 --- a/app/services/ci/authorize_job_artifact_service.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Ci - class AuthorizeJobArtifactService - include Gitlab::Utils::StrongMemoize - - # Max size of the zipped LSIF artifact - LSIF_ARTIFACT_MAX_SIZE = 20.megabytes - LSIF_ARTIFACT_TYPE = 'lsif' - - def initialize(job, params, max_size:) - @job = job - @max_size = max_size - @size = params[:filesize] - @type = params[:artifact_type].to_s - end - - def forbidden? - lsif? && !code_navigation_enabled? - end - - def too_large? - size && max_size <= size.to_i - end - - def headers - default_headers = JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_size) - default_headers.tap do |h| - h[:ProcessLsif] = true if lsif? && code_navigation_enabled? - end - end - - private - - attr_reader :job, :size, :type - - def code_navigation_enabled? - strong_memoize(:code_navigation_enabled) do - Feature.enabled?(:code_navigation, job.project, default_enabled: true) - end - end - - def lsif? - strong_memoize(:lsif) do - type == LSIF_ARTIFACT_TYPE - end - end - - def max_size - lsif? ? LSIF_ARTIFACT_MAX_SIZE : @max_size.to_i - end - end -end diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb index f0ffe67510b..9a6e103e5dd 100644 --- a/app/services/ci/create_job_artifacts_service.rb +++ b/app/services/ci/create_job_artifacts_service.rb @@ -3,42 +3,104 @@ module Ci class CreateJobArtifactsService < ::BaseService ArtifactsExistError = Class.new(StandardError) + + LSIF_ARTIFACT_TYPE = 'lsif' + OBJECT_STORAGE_ERRORS = [ Errno::EIO, Google::Apis::ServerError, Signet::RemoteServerError ].freeze - def execute(job, artifacts_file, params, metadata_file: nil) - return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file) + def initialize(job) + @job = job + @project = job.project + end + + def authorize(artifact_type:, filesize: nil) + result = validate_requirements(artifact_type: artifact_type, filesize: filesize) + return result unless result[:status] == :success + + headers = JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_size(artifact_type)) - artifact, artifact_metadata = build_artifact(job, artifacts_file, params, metadata_file) - result = parse_artifact(job, artifact) + if lsif?(artifact_type) + headers[:ProcessLsif] = true + headers[:ProcessLsifReferences] = Feature.enabled?(:code_navigation_references, project, default_enabled: false) + end + success(headers: headers) + end + + def execute(artifacts_file, params, metadata_file: nil) + result = validate_requirements(artifact_type: params[:artifact_type], filesize: artifacts_file.size) return result unless result[:status] == :success - persist_artifact(job, artifact, artifact_metadata) + return success if sha256_matches_existing_artifact?(params[:artifact_type], artifacts_file) + + artifact, artifact_metadata = build_artifact(artifacts_file, params, metadata_file) + result = parse_artifact(artifact) + + return result unless result[:status] == :success + + persist_artifact(artifact, artifact_metadata, params) end private - def build_artifact(job, artifacts_file, params, metadata_file) + attr_reader :job, :project + + def validate_requirements(artifact_type:, filesize:) + return forbidden_type_error(artifact_type) if forbidden_type?(artifact_type) + return too_large_error if too_large?(artifact_type, filesize) + + success + end + + def forbidden_type?(type) + lsif?(type) && !code_navigation_enabled? + end + + def too_large?(type, size) + size > max_size(type) if size + end + + def code_navigation_enabled? + Feature.enabled?(:code_navigation, project, default_enabled: true) + end + + def lsif?(type) + type == LSIF_ARTIFACT_TYPE + end + + def max_size(type) + Ci::JobArtifact.max_artifact_size(type: type, project: project) + end + + def forbidden_type_error(type) + error("#{type} artifacts are forbidden", :forbidden) + end + + def too_large_error + error('file size has reached maximum size limit', :payload_too_large) + end + + def build_artifact(artifacts_file, params, metadata_file) expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in artifact = Ci::JobArtifact.new( job_id: job.id, - project: job.project, + project: project, file: artifacts_file, - file_type: params['artifact_type'], - file_format: params['artifact_format'], + file_type: params[:artifact_type], + file_format: params[:artifact_format], file_sha256: artifacts_file.sha256, expire_in: expire_in) artifact_metadata = if metadata_file Ci::JobArtifact.new( job_id: job.id, - project: job.project, + project: project, file: metadata_file, file_type: :metadata, file_format: :gzip, @@ -46,31 +108,25 @@ module Ci expire_in: expire_in) end - if Feature.enabled?(:keep_latest_artifact_for_ref, job.project) - artifact.locked = true - artifact_metadata&.locked = true - end - [artifact, artifact_metadata] end - def parse_artifact(job, artifact) - unless Feature.enabled?(:ci_synchronous_artifact_parsing, job.project, default_enabled: true) + def parse_artifact(artifact) + unless Feature.enabled?(:ci_synchronous_artifact_parsing, project, default_enabled: true) return success end case artifact.file_type - when 'dotenv' then parse_dotenv_artifact(job, artifact) - when 'cluster_applications' then parse_cluster_applications_artifact(job, artifact) + when 'dotenv' then parse_dotenv_artifact(artifact) + when 'cluster_applications' then parse_cluster_applications_artifact(artifact) else success end end - def persist_artifact(job, artifact, artifact_metadata) + def persist_artifact(artifact, artifact_metadata, params) Ci::JobArtifact.transaction do artifact.save! artifact_metadata&.save! - unlock_previous_artifacts!(artifact) # NOTE: The `artifacts_expire_at` column is already deprecated and to be removed in the near future. job.update_column(:artifacts_expire_at, artifact.expire_at) @@ -78,42 +134,36 @@ module Ci success rescue ActiveRecord::RecordNotUnique => error - track_exception(error, job, params) + track_exception(error, params) error('another artifact of the same type already exists', :bad_request) rescue *OBJECT_STORAGE_ERRORS => error - track_exception(error, job, params) + track_exception(error, params) error(error.message, :service_unavailable) rescue => error - track_exception(error, job, params) + track_exception(error, params) error(error.message, :bad_request) end - def unlock_previous_artifacts!(artifact) - return unless Feature.enabled?(:keep_latest_artifact_for_ref, artifact.job.project) - - Ci::JobArtifact.for_ref(artifact.job.ref, artifact.project_id).locked.update_all(locked: false) - end - - def sha256_matches_existing_artifact?(job, artifact_type, artifacts_file) + def sha256_matches_existing_artifact?(artifact_type, artifacts_file) existing_artifact = job.job_artifacts.find_by_file_type(artifact_type) return false unless existing_artifact existing_artifact.file_sha256 == artifacts_file.sha256 end - def track_exception(error, job, params) + def track_exception(error, params) Gitlab::ErrorTracking.track_exception(error, job_id: job.id, project_id: job.project_id, - uploading_type: params['artifact_type'] + uploading_type: params[:artifact_type] ) end - def parse_dotenv_artifact(job, artifact) - Ci::ParseDotenvArtifactService.new(job.project, current_user).execute(artifact) + def parse_dotenv_artifact(artifact) + Ci::ParseDotenvArtifactService.new(project, current_user).execute(artifact) end - def parse_cluster_applications_artifact(job, artifact) + def parse_cluster_applications_artifact(artifact) Clusters::ParseClusterApplicationsArtifactService.new(job, job.user).execute(artifact) end end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 922c3556362..2d7f5014aa9 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -23,6 +23,24 @@ module Ci Gitlab::Ci::Pipeline::Chain::Limit::Activity, Gitlab::Ci::Pipeline::Chain::Limit::JobActivity].freeze + # Create a new pipeline in the specified project. + # + # @param [Symbol] source What event (Ci::Pipeline.sources) triggers the pipeline + # creation. + # @param [Boolean] ignore_skip_ci Whether skipping a pipeline creation when `[skip ci]` comment + # is present in the commit body + # @param [Boolean] save_on_errors Whether persisting an invalid pipeline when it encounters an + # error during creation (e.g. invalid yaml) + # @param [Ci::TriggerRequest] trigger_request The pipeline trigger triggers the pipeline creation. + # @param [Ci::PipelineSchedule] schedule The pipeline schedule triggers the pipeline creation. + # @param [MergeRequest] merge_request The merge request triggers the pipeline creation. + # @param [ExternalPullRequest] external_pull_request The external pull request triggers the pipeline creation. + # @param [Ci::Bridge] bridge The bridge job that triggers the downstream pipeline creation. + # @param [String] content The content of .gitlab-ci.yml to override the default config + # contents (e.g. .gitlab-ci.yml in repostiry). Mainly used for + # generating a dangling pipeline. + # + # @return [Ci::Pipeline] The created Ci::Pipeline object. # rubocop: disable Metrics/ParameterLists def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, external_pull_request: nil, bridge: nil, **options, &block) @pipeline = Ci::Pipeline.new @@ -77,7 +95,7 @@ module Ci def execute!(*args, &block) execute(*args, &block).tap do |pipeline| unless pipeline.persisted? - raise CreateError, pipeline.error_messages + raise CreateError, pipeline.full_error_messages end end end @@ -122,13 +140,8 @@ module Ci end end - def extra_options(options = {}) - # In Ruby 2.4, even when options is empty, f(**options) doesn't work when f - # doesn't have any parameters. We reproduce the Ruby 2.5 behavior by - # checking explicitly that no arguments are given. - raise ArgumentError if options.any? - - {} # overridden in EE + def extra_options(content: nil) + { content: content } end end end diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb index 5deb84812ac..1fa8926faa1 100644 --- a/app/services/ci/destroy_expired_job_artifacts_service.rb +++ b/app/services/ci/destroy_expired_job_artifacts_service.rb @@ -28,7 +28,7 @@ module Ci private def destroy_batch - artifact_batch = if Feature.enabled?(:keep_latest_artifact_for_ref) + artifact_batch = if Gitlab::Ci::Features.destroy_only_unlocked_expired_artifacts_enabled? Ci::JobArtifact.expired(BATCH_SIZE).unlocked else Ci::JobArtifact.expired(BATCH_SIZE) diff --git a/app/services/ci/pipeline_processing/atomic_processing_service.rb b/app/services/ci/pipeline_processing/atomic_processing_service.rb index b01a9d2e3b8..a23d5d8941a 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service.rb @@ -77,7 +77,7 @@ module Ci def update_processable!(processable) status = processable_status(processable) - return unless HasStatus::COMPLETED_STATUSES.include?(status) + return unless Ci::HasStatus::COMPLETED_STATUSES.include?(status) # transition status if possible Gitlab::OptimisticLocking.retry_lock(processable) do |subject| diff --git a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb index 2228328882d..d0aa8b04775 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb @@ -80,7 +80,7 @@ module Ci # TODO: This is hack to support # the same exact behaviour for Atomic and Legacy processing # that DAG is blocked from executing if dependent is not "complete" - if dag && statuses.any? { |status| HasStatus::COMPLETED_STATUSES.exclude?(status[:status]) } + if dag && statuses.any? { |status| Ci::HasStatus::COMPLETED_STATUSES.exclude?(status[:status]) } return 'pending' end diff --git a/app/services/ci/pipeline_processing/legacy_processing_service.rb b/app/services/ci/pipeline_processing/legacy_processing_service.rb index c471f7f0011..56fbc7271da 100644 --- a/app/services/ci/pipeline_processing/legacy_processing_service.rb +++ b/app/services/ci/pipeline_processing/legacy_processing_service.rb @@ -35,7 +35,7 @@ module Ci def process_stage_for_stage_scheduling(index) current_status = status_for_prior_stages(index) - return unless HasStatus::COMPLETED_STATUSES.include?(current_status) + return unless Ci::HasStatus::COMPLETED_STATUSES.include?(current_status) created_stage_scheduled_processables_in_stage(index).find_each.select do |build| process_build(build, current_status) @@ -73,7 +73,7 @@ module Ci def process_dag_build_with_needs(build) current_status = status_for_build_needs(build.needs.map(&:name)) - return unless HasStatus::COMPLETED_STATUSES.include?(current_status) + return unless Ci::HasStatus::COMPLETED_STATUSES.include?(current_status) process_build(build, current_status) end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 80ebe5f5eb6..1f24dce0458 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -9,6 +9,8 @@ module Ci end def execute(trigger_build_ids = nil, initial_process: false) + increment_processing_counter + update_retried if ::Gitlab::Ci::Features.atomic_processing?(pipeline.project) @@ -22,6 +24,10 @@ module Ci end end + def metrics + @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new + end + private # This method is for compatibility and data consistency and should be removed with 9.3 version of GitLab @@ -43,5 +49,9 @@ module Ci .update_all(retried: true) if latest_statuses.any? end # rubocop: enable CodeReuse/ActiveRecord + + def increment_processing_counter + metrics.pipeline_processing_events_counter.increment + end end end diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 17b9e56636b..3797ea1d96c 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -11,7 +11,7 @@ module Ci METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'.freeze DEFAULT_METRICS_SHARD = 'default'.freeze - Result = Struct.new(:build, :valid?) + Result = Struct.new(:build, :build_json, :valid?) def initialize(runner) @runner = runner @@ -59,7 +59,7 @@ module Ci end register_failure - Result.new(nil, valid) + Result.new(nil, nil, valid) end # rubocop: enable CodeReuse/ActiveRecord @@ -71,7 +71,7 @@ module Ci # In case when 2 runners try to assign the same build, second runner will be declined # with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method. if assign_runner!(build, params) - Result.new(build, true) + present_build!(build) end rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError # We are looping to find another build that is not conflicting @@ -83,8 +83,10 @@ module Ci # In case we hit the concurrency-access lock, # we still have to return 409 in the end, # to make sure that this is properly handled by runner. - Result.new(nil, false) + Result.new(nil, nil, false) rescue => ex + # If an error (e.g. GRPC::DeadlineExceeded) occurred constructing + # the result, consider this as a failure to be retried. scheduler_failure!(build) track_exception_for_build(ex, build) @@ -92,6 +94,15 @@ module Ci nil end + # Force variables evaluation to occur now + def present_build!(build) + # We need to use the presenter here because Gitaly calls in the presenter + # may fail, and we need to ensure the response has been generated. + presented_build = ::Ci::BuildRunnerPresenter.new(build) # rubocop:disable CodeReuse/Presenter + build_json = ::API::Entities::JobRequest::Response.new(presented_build).to_json + Result.new(build, build_json, true) + end + def assign_runner!(build, params) build.runner_id = runner.id build.runner_session_attributes = params[:session] if params[:session].present? diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index 23507a31c72..60b3d28b0c5 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -34,10 +34,6 @@ module Ci attributes[:user] = current_user - # TODO: we can probably remove this logic - # see: https://gitlab.com/gitlab-org/gitlab/-/issues/217930 - attributes[:scheduling_type] ||= build.find_legacy_scheduling_type - Ci::Build.transaction do # mark all other builds of that name as retried build.pipeline.builds.latest @@ -59,7 +55,9 @@ module Ci build = project.builds.new(attributes) build.assign_attributes(::Gitlab::Ci::Pipeline::Seed::Build.environment_attributes_for(build)) build.retried = false - build.save! + BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do + build.save! + end build end end diff --git a/app/services/ci/unlock_artifacts_service.rb b/app/services/ci/unlock_artifacts_service.rb new file mode 100644 index 00000000000..07faf90dd6d --- /dev/null +++ b/app/services/ci/unlock_artifacts_service.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Ci + class UnlockArtifactsService < ::BaseService + BATCH_SIZE = 100 + + def execute(ci_ref, before_pipeline = nil) + query = <<~SQL.squish + UPDATE "ci_pipelines" + SET "locked" = #{::Ci::Pipeline.lockeds[:unlocked]} + WHERE "ci_pipelines"."id" in ( + #{collect_pipelines(ci_ref, before_pipeline).select(:id).to_sql} + LIMIT #{BATCH_SIZE} + FOR UPDATE SKIP LOCKED + ) + RETURNING "ci_pipelines"."id"; + SQL + + loop do + break if ActiveRecord::Base.connection.exec_query(query).empty? + end + end + + private + + def collect_pipelines(ci_ref, before_pipeline) + pipeline_scope = ci_ref.pipelines + pipeline_scope = pipeline_scope.before_pipeline(before_pipeline) if before_pipeline + + pipeline_scope.artifacts_locked + end + end +end diff --git a/app/services/clusters/create_service.rb b/app/services/clusters/create_service.rb index 7b5bf6b32c2..6693a58683f 100644 --- a/app/services/clusters/create_service.rb +++ b/app/services/clusters/create_service.rb @@ -19,10 +19,6 @@ module Clusters cluster = Clusters::Cluster.new(cluster_params) - unless can_create_cluster? - cluster.errors.add(:base, _('Instance does not support multiple Kubernetes clusters')) - end - validate_management_project_permissions(cluster) return cluster if cluster.errors.present? @@ -55,16 +51,9 @@ module Clusters end end - # EE would override this method - def can_create_cluster? - clusterable.clusters.empty? - end - def validate_management_project_permissions(cluster) Clusters::Management::ValidateManagementProjectPermissionsService.new(current_user) .execute(cluster, params[:management_project_id]) end end end - -Clusters::CreateService.prepend_if_ee('EE::Clusters::CreateService') diff --git a/app/services/clusters/parse_cluster_applications_artifact_service.rb b/app/services/clusters/parse_cluster_applications_artifact_service.rb index 35fba5f47c7..6a0ca0ef9d0 100644 --- a/app/services/clusters/parse_cluster_applications_artifact_service.rb +++ b/app/services/clusters/parse_cluster_applications_artifact_service.rb @@ -5,7 +5,7 @@ module Clusters include Gitlab::Utils::StrongMemoize MAX_ACCEPTABLE_ARTIFACT_SIZE = 5.kilobytes - RELEASE_NAMES = %w[prometheus].freeze + RELEASE_NAMES = %w[prometheus cilium].freeze def initialize(job, current_user) @job = job diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb index 4678d051d29..a58e9aefcec 100644 --- a/app/services/concerns/exclusive_lease_guard.rb +++ b/app/services/concerns/exclusive_lease_guard.rb @@ -21,7 +21,7 @@ module ExclusiveLeaseGuard lease = exclusive_lease.try_obtain unless lease - log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.') + log_error("Cannot obtain an exclusive lease for #{self.class.name}. There must be another instance already in execution.") return end diff --git a/app/services/concerns/incident_management/settings.rb b/app/services/concerns/incident_management/settings.rb index 5f56d6e7f53..491bd4fa6bf 100644 --- a/app/services/concerns/incident_management/settings.rb +++ b/app/services/concerns/incident_management/settings.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module IncidentManagement module Settings + include Gitlab::Utils::StrongMemoize + def incident_management_setting strong_memoize(:incident_management_setting) do project.incident_management_setting || diff --git a/app/services/deploy_keys/collect_keys_service.rb b/app/services/deploy_keys/collect_keys_service.rb new file mode 100644 index 00000000000..2ef49bf0f30 --- /dev/null +++ b/app/services/deploy_keys/collect_keys_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module DeployKeys + class CollectKeysService + def initialize(project, current_user) + @project = project + @current_user = current_user + end + + def execute + return [] unless current_user && project && user_can_read_project + + project.deploy_keys_projects + .with_deploy_keys + .with_write_access + .map(&:deploy_key) + end + + private + + def user_can_read_project + Ability.allowed?(current_user, :read_project, project) + end + + attr_reader :project, :current_user + end +end diff --git a/app/services/event_create_service.rb b/app/services/event_create_service.rb index 89c3225dbcd..ad36fe70b3a 100644 --- a/app/services/event_create_service.rb +++ b/app/services/event_create_service.rb @@ -11,44 +11,30 @@ class EventCreateService IllegalActionError = Class.new(StandardError) def open_issue(issue, current_user) - create_resource_event(issue, current_user, :opened) - create_record_event(issue, current_user, :created) end def close_issue(issue, current_user) - create_resource_event(issue, current_user, :closed) - create_record_event(issue, current_user, :closed) end def reopen_issue(issue, current_user) - create_resource_event(issue, current_user, :reopened) - create_record_event(issue, current_user, :reopened) end def open_mr(merge_request, current_user) - create_resource_event(merge_request, current_user, :opened) - create_record_event(merge_request, current_user, :created) end def close_mr(merge_request, current_user) - create_resource_event(merge_request, current_user, :closed) - create_record_event(merge_request, current_user, :closed) end def reopen_mr(merge_request, current_user) - create_resource_event(merge_request, current_user, :reopened) - create_record_event(merge_request, current_user, :reopened) end def merge_mr(merge_request, current_user) - create_resource_event(merge_request, current_user, :merged) - create_record_event(merge_request, current_user, :merged) end @@ -97,23 +83,13 @@ class EventCreateService end def save_designs(current_user, create: [], update: []) - created = create.group_by(&:project).flat_map do |project, designs| - Feature.enabled?(:design_activity_events, project) ? designs : [] - end.to_set - updated = update.group_by(&:project).flat_map do |project, designs| - Feature.enabled?(:design_activity_events, project) ? designs : [] - end.to_set - return [] if created.empty? && updated.empty? - - records = created.zip([:created].cycle) + updated.zip([:updated].cycle) + records = create.zip([:created].cycle) + update.zip([:updated].cycle) + return [] if records.empty? create_record_events(records, current_user) end def destroy_designs(designs, current_user) - designs = designs.select do |design| - Feature.enabled?(:design_activity_events, design.project) - end return [] unless designs.present? create_record_events(designs.zip([:destroyed].cycle), current_user) @@ -127,8 +103,6 @@ class EventCreateService # # @return a tuple of event and either :found or :created def wiki_event(wiki_page_meta, author, action) - return unless Feature.enabled?(:wiki_events) - raise IllegalActionError, action unless Event::WIKI_ACTIONS.include?(action) if duplicate = existing_wiki_event(wiki_page_meta, action) @@ -142,9 +116,15 @@ class EventCreateService event.update_columns(updated_at: time_stamp, created_at: time_stamp) end + Gitlab::UsageDataCounters::TrackUniqueActions.track_action(event_action: action, event_target: wiki_page_meta.class, author_id: author.id) + event end + def approve_mr(merge_request, current_user) + create_record_event(merge_request, current_user, :approved) + end + private def existing_wiki_event(wiki_page_meta, action) @@ -182,7 +162,13 @@ class EventCreateService .merge(action: action, target_id: record.id, target_type: record.class.name) end - Event.insert_all(attribute_sets, returning: %w[id]) + result = Event.insert_all(attribute_sets, returning: %w[id]) + + pairs.each do |record, status| + Gitlab::UsageDataCounters::TrackUniqueActions.track_action(event_action: status, event_target: record.class, author_id: current_user.id) + end + + result end def create_push_event(service_class, project, current_user, push_data) @@ -197,6 +183,8 @@ class EventCreateService new_event end + Gitlab::UsageDataCounters::TrackUniqueActions.track_action(event_action: :pushed, event_target: Project, author_id: current_user.id) + Users::LastPushEventService.new(current_user) .cache_last_push_event(event) @@ -225,18 +213,6 @@ class EventCreateService { resource_parent_attr => resource_parent.id } end - - def create_resource_event(issuable, current_user, status) - return unless state_change_tracking_enabled?(issuable) - - ResourceEvents::ChangeStateService.new(resource: issuable, user: current_user) - .execute(status) - end - - def state_change_tracking_enabled?(issuable) - issuable&.respond_to?(:resource_state_events) && - ::Feature.enabled?(:track_resource_state_change_events, issuable&.project) - end end EventCreateService.prepend_if_ee('EE::EventCreateService') diff --git a/app/services/files/base_service.rb b/app/services/files/base_service.rb index 39e614d6569..d42f718a272 100644 --- a/app/services/files/base_service.rb +++ b/app/services/files/base_service.rb @@ -25,7 +25,7 @@ module Files return false unless commit_id last_commit = Gitlab::Git::Commit - .last_for_path(@start_project.repository, @start_branch, path) + .last_for_path(@start_project.repository, @start_branch, path, literal_pathspec: true) return false unless last_commit diff --git a/app/services/git/branch_push_service.rb b/app/services/git/branch_push_service.rb index 5c1ee981d0c..2ec6ac99ece 100644 --- a/app/services/git/branch_push_service.rb +++ b/app/services/git/branch_push_service.rb @@ -29,6 +29,7 @@ module Git perform_housekeeping stop_environments + unlock_artifacts true end @@ -60,6 +61,12 @@ module Git Ci::StopEnvironmentsService.new(project, current_user).execute(branch_name) end + def unlock_artifacts + return unless removing_branch? + + Ci::RefDeleteUnlockArtifactsWorker.perform_async(project.id, current_user.id, ref) + end + def execute_related_hooks BranchHooksService.new(project, current_user, params).execute end diff --git a/app/services/git/tag_push_service.rb b/app/services/git/tag_push_service.rb index 9a266f7d74c..120c4cde94b 100644 --- a/app/services/git/tag_push_service.rb +++ b/app/services/git/tag_push_service.rb @@ -10,7 +10,25 @@ module Git project.repository.before_push_tag TagHooksService.new(project, current_user, params).execute + unlock_artifacts + true end + + private + + def unlock_artifacts + return unless removing_tag? + + Ci::RefDeleteUnlockArtifactsWorker.perform_async(project.id, current_user.id, ref) + end + + def removing_tag? + Gitlab::Git.blank_ref?(newrev) + end + + def tag_name + Gitlab::Git.ref_name(ref) + end end end diff --git a/app/services/git/wiki_push_service.rb b/app/services/git/wiki_push_service.rb index 8bdbc28f3e8..b3937a10a70 100644 --- a/app/services/git/wiki_push_service.rb +++ b/app/services/git/wiki_push_service.rb @@ -23,7 +23,7 @@ module Git end def can_process_wiki_events? - Feature.enabled?(:wiki_events) && Feature.enabled?(:wiki_events_on_git_push, project) + Feature.enabled?(:wiki_events_on_git_push, project) end def push_changes diff --git a/app/services/gpg_keys/destroy_service.rb b/app/services/gpg_keys/destroy_service.rb new file mode 100644 index 00000000000..cecbfe26611 --- /dev/null +++ b/app/services/gpg_keys/destroy_service.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module GpgKeys + class DestroyService < Keys::BaseService + def execute(key) + key.destroy + end + end +end diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb index eb1b8d4fcc0..ce583095168 100644 --- a/app/services/groups/create_service.rb +++ b/app/services/groups/create_service.rb @@ -28,7 +28,11 @@ module Groups @group.build_chat_team(name: response['name'], team_id: response['id']) end - @group.add_owner(current_user) if @group.save + if @group.save + @group.add_owner(current_user) + add_settings_record + end + @group end @@ -79,6 +83,10 @@ module Groups params[:visibility_level] = Gitlab::CurrentSettings.current_application_settings.default_group_visibility end + + def add_settings_record + @group.create_namespace_settings + end end end diff --git a/app/services/groups/update_shared_runners_service.rb b/app/services/groups/update_shared_runners_service.rb new file mode 100644 index 00000000000..63f57104510 --- /dev/null +++ b/app/services/groups/update_shared_runners_service.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Groups + class UpdateSharedRunnersService < Groups::BaseService + def execute + return error('Operation not allowed', 403) unless can?(current_user, :admin_group, group) + + validate_params + + enable_or_disable_shared_runners! + allow_or_disallow_descendants_override_disabled_shared_runners! + + success + + rescue Group::UpdateSharedRunnersError => error + error(error.message) + end + + private + + def validate_params + if Gitlab::Utils.to_boolean(params[:shared_runners_enabled]) && !params[:allow_descendants_override_disabled_shared_runners].nil? + raise Group::UpdateSharedRunnersError, 'Cannot set shared_runners_enabled to true and allow_descendants_override_disabled_shared_runners' + end + end + + def enable_or_disable_shared_runners! + return if params[:shared_runners_enabled].nil? + + if Gitlab::Utils.to_boolean(params[:shared_runners_enabled]) + group.enable_shared_runners! + else + group.disable_shared_runners! + end + end + + def allow_or_disallow_descendants_override_disabled_shared_runners! + return if params[:allow_descendants_override_disabled_shared_runners].nil? + + # Needs to reset group because if both params are present could result in error + group.reset + + if Gitlab::Utils.to_boolean(params[:allow_descendants_override_disabled_shared_runners]) + group.allow_descendants_override_disabled_shared_runners! + else + group.disallow_descendants_override_disabled_shared_runners! + end + end + end +end diff --git a/app/services/import/bitbucket_server_service.rb b/app/services/import/bitbucket_server_service.rb new file mode 100644 index 00000000000..86e8215821e --- /dev/null +++ b/app/services/import/bitbucket_server_service.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Import + class BitbucketServerService < Import::BaseService + attr_reader :client, :params, :current_user + + def execute(credentials) + if blocked_url? + return log_and_return_error("Invalid URL: #{url}", :bad_request) + end + + unless authorized? + return log_and_return_error("You don't have permissions to create this project", :unauthorized) + end + + unless repo + return log_and_return_error("Project %{project_repo} could not be found" % { project_repo: "#{project_key}/#{repo_slug}" }, :unprocessable_entity) + end + + project = create_project(credentials) + + if project.persisted? + success(project) + else + log_and_return_error(project_save_error(project), :unprocessable_entity) + end + rescue BitbucketServer::Connection::ConnectionError => e + log_and_return_error("Import failed due to a BitBucket Server error: #{e}", :bad_request) + end + + private + + def create_project(credentials) + Gitlab::BitbucketServerImport::ProjectCreator.new( + project_key, + repo_slug, + repo, + project_name, + target_namespace, + current_user, + credentials + ).execute + end + + def repo + @repo ||= client.repo(project_key, repo_slug) + end + + def project_name + @project_name ||= params[:new_name].presence || repo.name + end + + def namespace_path + @namespace_path ||= params[:new_namespace].presence || current_user.namespace_path + end + + def target_namespace + @target_namespace ||= find_or_create_namespace(namespace_path, current_user.namespace_path) + end + + def repo_slug + @repo_slug ||= params[:bitbucket_server_repo] + end + + def project_key + @project_key ||= params[:bitbucket_server_project] + end + + def url + @url ||= params[:bitbucket_server_url] + end + + def authorized? + can?(current_user, :create_projects, target_namespace) + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def blocked_url? + Gitlab::UrlBlocker.blocked_url?( + url, + { + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests?, + schemes: %w(http https) + } + ) + end + + def log_and_return_error(message, error_type) + log_error(message) + error(_(message), error_type) + end + + def log_error(message) + Gitlab::Import::Logger.error( + message: 'Import failed due to a BitBucket Server error', + error: message + ) + end + end +end diff --git a/app/services/incident_management/create_incident_label_service.rb b/app/services/incident_management/create_incident_label_service.rb new file mode 100644 index 00000000000..dbd0d78fa3c --- /dev/null +++ b/app/services/incident_management/create_incident_label_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module IncidentManagement + class CreateIncidentLabelService < BaseService + LABEL_PROPERTIES = { + title: 'incident', + color: '#CC0033', + description: <<~DESCRIPTION.chomp + Denotes a disruption to IT services and \ + the associated issues require immediate attention + DESCRIPTION + }.freeze + + def execute + label = Labels::FindOrCreateService + .new(current_user, project, **LABEL_PROPERTIES) + .execute + + if label.invalid? + log_invalid_label_info(label) + return ServiceResponse.error(payload: { label: label }, message: full_error_message(label)) + end + + ServiceResponse.success(payload: { label: label }) + end + + private + + def log_invalid_label_info(label) + log_info <<~TEXT.chomp + Cannot create incident label "#{label.title}" \ + for "#{label.project.full_name}": #{full_error_message(label)}. + TEXT + end + + def full_error_message(label) + label.errors.full_messages.to_sentence + end + end +end diff --git a/app/services/incident_management/create_issue_service.rb b/app/services/incident_management/create_issue_service.rb index 4b59dc64cec..5e1e0863115 100644 --- a/app/services/incident_management/create_issue_service.rb +++ b/app/services/incident_management/create_issue_service.rb @@ -4,21 +4,12 @@ module IncidentManagement class CreateIssueService < BaseService include Gitlab::Utils::StrongMemoize - INCIDENT_LABEL = { - title: 'incident', - color: '#CC0033', - description: <<~DESCRIPTION.chomp - Denotes a disruption to IT services and \ - the associated issues require immediate attention - DESCRIPTION - }.freeze - - def initialize(project, params, user = User.alert_bot) - super(project, user, params) + def initialize(project, params) + super(project, User.alert_bot, params) end - def execute(skip_settings_check: false) - return error_with('setting disabled') unless skip_settings_check || incident_management_setting.create_issue? + def execute + return error_with('setting disabled') unless incident_management_setting.create_issue? return error_with('invalid alert') unless alert.valid? issue = create_issue @@ -30,26 +21,19 @@ module IncidentManagement private def create_issue - issue = do_create_issue(label_ids: issue_label_ids) + label_result = find_or_create_incident_label - # Create an unlabelled issue if we couldn't create the issue - # due to labels errors. + # Create an unlabelled issue if we couldn't create the label + # due to a race condition. # See https://gitlab.com/gitlab-org/gitlab-foss/issues/65042 - if issue.errors.include?(:labels) - log_label_error(issue) - issue = do_create_issue - end - - issue - end + extra_params = label_result.success? ? { label_ids: [label_result.payload[:label].id] } : {} - def do_create_issue(**params) Issues::CreateService.new( project, current_user, title: issue_title, description: issue_description, - **params + **extra_params ).execute end @@ -67,16 +51,8 @@ module IncidentManagement ].compact.join(horizontal_line) end - def issue_label_ids - [ - find_or_create_label(**INCIDENT_LABEL) - ].compact.map(&:id) - end - - def find_or_create_label(**params) - Labels::FindOrCreateService - .new(current_user, project, **params) - .execute + def find_or_create_incident_label + IncidentManagement::CreateIncidentLabelService.new(project, current_user).execute end def alert_summary @@ -108,15 +84,6 @@ module IncidentManagement issue.errors.full_messages.to_sentence end - def log_label_error(issue) - log_info <<~TEXT.chomp - Cannot create incident issue with labels \ - #{issue.labels.map(&:title).inspect} \ - for "#{project.full_name}": #{issue.errors.full_messages.to_sentence}. - Retrying without labels. - TEXT - end - def error_with(message) log_error(%{Cannot create incident issue for "#{project.full_name}": #{message}}) diff --git a/app/services/incident_management/pager_duty/create_incident_issue_service.rb b/app/services/incident_management/pager_duty/create_incident_issue_service.rb new file mode 100644 index 00000000000..ee0feb49e0d --- /dev/null +++ b/app/services/incident_management/pager_duty/create_incident_issue_service.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module IncidentManagement + module PagerDuty + class CreateIncidentIssueService < BaseService + include IncidentManagement::Settings + + def initialize(project, incident_payload) + super(project, User.alert_bot, incident_payload) + end + + def execute + return forbidden unless webhook_available? + + issue = create_issue + return error(issue.errors.full_messages.to_sentence, issue) unless issue.valid? + + success(issue) + end + + private + + alias_method :incident_payload, :params + + def create_issue + label_result = find_or_create_incident_label + + # Create an unlabelled issue if we couldn't create the label + # due to a race condition. + # See https://gitlab.com/gitlab-org/gitlab-foss/issues/65042 + extra_params = label_result.success? ? { label_ids: [label_result.payload[:label].id] } : {} + + Issues::CreateService.new( + project, + current_user, + title: issue_title, + description: issue_description, + **extra_params + ).execute + end + + def webhook_available? + Feature.enabled?(:pagerduty_webhook, project) && + incident_management_setting.pagerduty_active? + end + + def forbidden + ServiceResponse.error(message: 'Forbidden', http_status: :forbidden) + end + + def find_or_create_incident_label + ::IncidentManagement::CreateIncidentLabelService.new(project, current_user).execute + end + + def issue_title + incident_payload['title'] + end + + def issue_description + Gitlab::IncidentManagement::PagerDuty::IncidentIssueDescription.new(incident_payload).to_s + end + + def success(issue) + ServiceResponse.success(payload: { issue: issue }) + end + + def error(message, issue = nil) + ServiceResponse.error(payload: { issue: issue }, message: message) + end + end + end +end diff --git a/app/services/incident_management/pager_duty/process_webhook_service.rb b/app/services/incident_management/pager_duty/process_webhook_service.rb new file mode 100644 index 00000000000..5dd3186694a --- /dev/null +++ b/app/services/incident_management/pager_duty/process_webhook_service.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module IncidentManagement + module PagerDuty + class ProcessWebhookService < BaseService + include Gitlab::Utils::StrongMemoize + include IncidentManagement::Settings + + # https://developer.pagerduty.com/docs/webhooks/webhook-behavior/#size-limit + PAGER_DUTY_PAYLOAD_SIZE_LIMIT = 55.kilobytes + + # https://developer.pagerduty.com/docs/webhooks/v2-overview/#webhook-types + PAGER_DUTY_PROCESSABLE_EVENT_TYPES = %w(incident.trigger).freeze + + def execute(token) + return forbidden unless webhook_setting_active? + return unauthorized unless valid_token?(token) + return bad_request unless valid_payload_size? + + process_incidents + + accepted + end + + private + + def process_incidents + pager_duty_processable_events.each do |event| + ::IncidentManagement::PagerDuty::ProcessIncidentWorker.perform_async(project.id, event['incident']) + end + end + + def pager_duty_processable_events + strong_memoize(:pager_duty_processable_events) do + ::PagerDuty::WebhookPayloadParser + .call(params.to_h) + .filter { |msg| msg['event'].in?(PAGER_DUTY_PROCESSABLE_EVENT_TYPES) } + end + end + + def webhook_setting_active? + Feature.enabled?(:pagerduty_webhook, project) && + incident_management_setting.pagerduty_active? + end + + def valid_token?(token) + token && incident_management_setting.pagerduty_token == token + end + + def valid_payload_size? + Gitlab::Utils::DeepSize.new(params, max_size: PAGER_DUTY_PAYLOAD_SIZE_LIMIT).valid? + end + + def accepted + ServiceResponse.success(http_status: :accepted) + end + + def forbidden + ServiceResponse.error(message: 'Forbidden', http_status: :forbidden) + end + + def unauthorized + ServiceResponse.error(message: 'Unauthorized', http_status: :unauthorized) + end + + def bad_request + ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) + end + end + end +end diff --git a/app/services/issuable/bulk_update_service.rb b/app/services/issuable/bulk_update_service.rb index 2902385da4a..79be771b3fb 100644 --- a/app/services/issuable/bulk_update_service.rb +++ b/app/services/issuable/bulk_update_service.rb @@ -11,40 +11,29 @@ module Issuable end def execute(type) - model_class = type.classify.constantize - update_class = type.classify.pluralize.constantize::UpdateService - ids = params.delete(:issuable_ids).split(",") - items = find_issuables(parent, model_class, ids) + set_update_params(type) + items = update_issuables(type, ids) + response_success(payload: { count: items.count }) + rescue ArgumentError => e + response_error(e.message, 422) + end + + private + + def set_update_params(type) params.slice!(*permitted_attrs(type)) params.delete_if { |k, v| v.blank? } if params[:assignee_ids] == [IssuableFinder::Params::NONE.to_s] params[:assignee_ids] = [] end - - items.each do |issuable| - next unless can?(current_user, :"update_#{type}", issuable) - - update_class.new(issuable.issuing_parent, current_user, params).execute(issuable) - end - - { - count: items.count, - success: !items.count.zero? - } end - private - def permitted_attrs(type) attrs = %i(state_event milestone_id add_label_ids remove_label_ids subscription_event) - issuable_specific_attrs(type, attrs) - end - - def issuable_specific_attrs(type, attrs) if type == 'issue' || type == 'merge_request' attrs.push(:assignee_ids) else @@ -52,6 +41,20 @@ module Issuable end end + def update_issuables(type, ids) + model_class = type.classify.constantize + update_class = type.classify.pluralize.constantize::UpdateService + items = find_issuables(parent, model_class, ids) + + items.each do |issuable| + next unless can?(current_user, :"update_#{type}", issuable) + + update_class.new(issuable.issuing_parent, current_user, params).execute(issuable) + end + + items + end + def find_issuables(parent, model_class, ids) if parent.is_a?(Project) model_class.id_in(ids).of_projects(parent) @@ -59,6 +62,14 @@ module Issuable model_class.id_in(ids).of_projects(parent.all_projects) end end + + def response_success(message: nil, payload: nil) + ServiceResponse.success(message: message, payload: payload) + end + + def response_error(message, http_status) + ServiceResponse.error(message: message, http_status: http_status) + end end end diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 38b10996f44..65a73dadc2e 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -97,29 +97,6 @@ class IssuableBaseService < BaseService params.delete(label_key) if params[label_key].nil? end - def filter_labels_in_param(key) - return if params[key].to_a.empty? - - params[key] = available_labels.id_in(params[key]).pluck_primary_key - end - - def find_or_create_label_ids - labels = params.delete(:labels) - - return unless labels - - params[:label_ids] = labels.map do |label_name| - label = Labels::FindOrCreateService.new( - current_user, - parent, - title: label_name.strip, - available_labels: available_labels - ).execute - - label.try(:id) - end.compact - end - def labels_service @labels_service ||= ::Labels::AvailableLabelsService.new(current_user, parent, params) end @@ -138,7 +115,7 @@ class IssuableBaseService < BaseService new_label_ids.uniq end - def handle_quick_actions_on_create(issuable) + def handle_quick_actions(issuable) merge_quick_actions_into_params!(issuable) end @@ -146,17 +123,21 @@ class IssuableBaseService < BaseService original_description = params.fetch(:description, issuable.description) description, command_params = - QuickActions::InterpretService.new(project, current_user) + QuickActions::InterpretService.new(project, current_user, quick_action_options) .execute(original_description, issuable, only: only) # Avoid a description already set on an issuable to be overwritten by a nil - params[:description] = description if description + params[:description] = description if description && description != original_description params.merge!(command_params) end + def quick_action_options + {} + end + def create(issuable) - handle_quick_actions_on_create(issuable) + handle_quick_actions(issuable) filter_params(issuable) params.delete(:state_event) @@ -200,11 +181,13 @@ class IssuableBaseService < BaseService end def update(issuable) + handle_quick_actions(issuable) + filter_params(issuable) + change_state(issuable) change_subscription(issuable) change_todo(issuable) toggle_award(issuable) - filter_params(issuable) old_associations = associations_before_update(issuable) label_ids = process_label_ids(params, existing_label_ids: issuable.label_ids) diff --git a/app/services/issues/move_service.rb b/app/services/issues/move_service.rb index 2409396c1ac..ce1466307e1 100644 --- a/app/services/issues/move_service.rb +++ b/app/services/issues/move_service.rb @@ -19,11 +19,22 @@ module Issues notify_participants + # Updates old issue sent notifications allowing + # to receive service desk emails on the new moved issue. + update_service_desk_sent_notifications + new_entity end private + def update_service_desk_sent_notifications + return unless original_entity.from_service_desk? + + original_entity + .sent_notifications.update_all(project_id: new_entity.project_id, noteable_id: new_entity.id) + end + def update_old_entity super diff --git a/app/services/jira/requests/base.rb b/app/services/jira/requests/base.rb index 7521c7610cb..7c6db372257 100644 --- a/app/services/jira/requests/base.rb +++ b/app/services/jira/requests/base.rb @@ -5,28 +5,32 @@ module Jira class Base include ProjectServicesLoggable - PER_PAGE = 50 + JIRA_API_VERSION = 2 - attr_reader :jira_service, :project, :limit, :start_at, :query - - def initialize(jira_service, limit: PER_PAGE, start_at: 0, query: nil) + def initialize(jira_service, params = {}) @project = jira_service&.project @jira_service = jira_service - - @limit = limit - @start_at = start_at - @query = query end def execute return ServiceResponse.error(message: _('Jira service not configured.')) unless jira_service&.active? - return ServiceResponse.success(payload: empty_payload) if limit.to_i <= 0 request end + def base_api_url + "/rest/api/#{api_version}" + end + private + attr_reader :jira_service, :project + + # override this method in the specific request class implementation if a differnt API version is required + def api_version + JIRA_API_VERSION + end + def client @client ||= jira_service.client end diff --git a/app/services/jira/requests/projects.rb b/app/services/jira/requests/projects.rb deleted file mode 100644 index da464503211..00000000000 --- a/app/services/jira/requests/projects.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Jira - module Requests - class Projects < Base - extend ::Gitlab::Utils::Override - - private - - override :url - def url - '/rest/api/2/project/search?query=%{query}&maxResults=%{limit}&startAt=%{start_at}' % - { query: CGI.escape(query.to_s), limit: limit.to_i, start_at: start_at.to_i } - end - - override :build_service_response - def build_service_response(response) - return ServiceResponse.success(payload: empty_payload) unless response['values'].present? - - ServiceResponse.success(payload: { projects: map_projects(response), is_last: response['isLast'] }) - end - - def map_projects(response) - response['values'].map { |v| JIRA::Resource::Project.build(client, v) } - end - - def empty_payload - { projects: [], is_last: true } - end - end - end -end diff --git a/app/services/jira/requests/projects/list_service.rb b/app/services/jira/requests/projects/list_service.rb new file mode 100644 index 00000000000..8ecfd358ffb --- /dev/null +++ b/app/services/jira/requests/projects/list_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Jira + module Requests + module Projects + class ListService < Base + extend ::Gitlab::Utils::Override + + def initialize(jira_service, params: {}) + super(jira_service, params) + + @query = params[:query] + end + + private + + attr_reader :query + + override :url + def url + "#{base_api_url}/project" + end + + override :build_service_response + def build_service_response(response) + return ServiceResponse.success(payload: empty_payload) unless response.present? + + ServiceResponse.success(payload: { projects: map_projects(response), is_last: true }) + end + + def map_projects(response) + response.map { |v| JIRA::Resource::Project.build(client, v) }.select(&method(:match_query?)) + end + + def match_query?(jira_project) + query = query.to_s.downcase + + jira_project&.key&.downcase&.include?(query) || jira_project&.name&.downcase&.include?(query) + end + + def empty_payload + { projects: [], is_last: true } + end + end + end + end +end diff --git a/app/services/jira_import/start_import_service.rb b/app/services/jira_import/start_import_service.rb index a06cc6df719..f85f686c61a 100644 --- a/app/services/jira_import/start_import_service.rb +++ b/app/services/jira_import/start_import_service.rb @@ -2,23 +2,39 @@ module JiraImport class StartImportService - attr_reader :user, :project, :jira_project_key + attr_reader :user, :project, :jira_project_key, :users_mapping - def initialize(user, project, jira_project_key) + def initialize(user, project, jira_project_key, users_mapping) @user = user @project = project @jira_project_key = jira_project_key + @users_mapping = users_mapping end def execute validation_response = validate return validation_response if validation_response&.error? + store_users_mapping create_and_schedule_import end private + def store_users_mapping + return if users_mapping.blank? + + mapping = users_mapping.map do |map| + next if !map[:jira_account_id] || !map[:gitlab_id] + + [map[:jira_account_id], map[:gitlab_id]] + end.compact.to_h + + return if mapping.blank? + + Gitlab::JiraImport.cache_users_mapping(project.id, mapping) + end + def create_and_schedule_import jira_import = build_jira_import project.import_type = 'jira' diff --git a/app/services/jira_import/users_mapper.rb b/app/services/jira_import/users_mapper.rb index 31a3f721556..c3cbeb157bd 100644 --- a/app/services/jira_import/users_mapper.rb +++ b/app/services/jira_import/users_mapper.rb @@ -14,9 +14,8 @@ module JiraImport { jira_account_id: jira_user['accountId'], jira_display_name: jira_user['displayName'], - jira_email: jira_user['emailAddress'], - gitlab_id: match_user(jira_user) - } + jira_email: jira_user['emailAddress'] + }.merge(match_user(jira_user)) end end @@ -25,7 +24,7 @@ module JiraImport # TODO: Matching user by email and displayName will be done as the part # of follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/219023 def match_user(jira_user) - nil + { gitlab_id: nil, gitlab_username: nil, gitlab_name: nil } end end end diff --git a/app/services/labels/available_labels_service.rb b/app/services/labels/available_labels_service.rb index 979964e09fd..3b226f39d04 100644 --- a/app/services/labels/available_labels_service.rb +++ b/app/services/labels/available_labels_service.rb @@ -34,7 +34,7 @@ module Labels return [] if ids.empty? # rubocop:disable CodeReuse/ActiveRecord - existing_ids = available_labels.by_ids(ids).pluck(:id) + existing_ids = available_labels.id_in(ids).pluck(:id) # rubocop:enable CodeReuse/ActiveRecord ids.map(&:to_i) & existing_ids end diff --git a/app/services/labels/transfer_service.rb b/app/services/labels/transfer_service.rb index e6f9cf35fcb..a05090d6bfb 100644 --- a/app/services/labels/transfer_service.rb +++ b/app/services/labels/transfer_service.rb @@ -15,14 +15,18 @@ module Labels def execute return unless old_group.present? + # rubocop: disable CodeReuse/ActiveRecord + link_ids = group_labels_applied_to_issues.pluck("label_links.id") + + group_labels_applied_to_merge_requests.pluck("label_links.id") + # rubocop: disable CodeReuse/ActiveRecord + Label.transaction do labels_to_transfer.find_each do |label| new_label_id = find_or_create_label!(label) next if new_label_id == label.id - update_label_links(group_labels_applied_to_issues, old_label_id: label.id, new_label_id: new_label_id) - update_label_links(group_labels_applied_to_merge_requests, old_label_id: label.id, new_label_id: new_label_id) + update_label_links(link_ids, old_label_id: label.id, new_label_id: new_label_id) update_label_priorities(old_label_id: label.id, new_label_id: new_label_id) end end @@ -46,20 +50,20 @@ module Labels # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_issues - Label.joins(:issues) + @group_labels_applied_to_issues ||= Label.joins(:issues) .where( issues: { project_id: project.id }, - labels: { type: 'GroupLabel', group_id: old_group.self_and_ancestors } + labels: { group_id: old_group.self_and_ancestors } ) end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def group_labels_applied_to_merge_requests - Label.joins(:merge_requests) + @group_labels_applied_to_merge_requests ||= Label.joins(:merge_requests) .where( merge_requests: { target_project_id: project.id }, - labels: { type: 'GroupLabel', group_id: old_group.self_and_ancestors } + labels: { group_id: old_group.self_and_ancestors } ) end # rubocop: enable CodeReuse/ActiveRecord @@ -72,14 +76,7 @@ module Labels end # rubocop: disable CodeReuse/ActiveRecord - def update_label_links(labels, old_label_id:, new_label_id:) - # use 'labels' relation to get label_link ids only of issues/MRs - # in the project being transferred. - # IDs are fetched in a separate query because MySQL doesn't - # allow referring of 'label_links' table in UPDATE query: - # https://gitlab.com/gitlab-org/gitlab-foss/-/jobs/62435068 - link_ids = labels.pluck('label_links.id') - + def update_label_links(link_ids, old_label_id:, new_label_id:) LabelLink.where(id: link_ids, label_id: old_label_id) .update_all(label_id: new_label_id) end diff --git a/app/services/members/create_service.rb b/app/services/members/create_service.rb index 0b729981a93..610288c5e76 100644 --- a/app/services/members/create_service.rb +++ b/app/services/members/create_service.rb @@ -22,7 +22,7 @@ module Members errors = [] members.each do |member| - if member.errors.any? + if member.invalid? current_error = # Invited users may not have an associated user if member.user.present? diff --git a/app/services/members/destroy_service.rb b/app/services/members/destroy_service.rb index 20f64a99ad7..fdd43260521 100644 --- a/app/services/members/destroy_service.rb +++ b/app/services/members/destroy_service.rb @@ -2,8 +2,8 @@ module Members class DestroyService < Members::BaseService - def execute(member, skip_authorization: false, skip_subresources: false) - raise Gitlab::Access::AccessDeniedError unless skip_authorization || can_destroy_member?(member) + def execute(member, skip_authorization: false, skip_subresources: false, unassign_issuables: false, destroy_bot: false) + raise Gitlab::Access::AccessDeniedError unless skip_authorization || authorized?(member, destroy_bot) @skip_auth = skip_authorization @@ -19,6 +19,7 @@ module Members delete_subresources(member) unless skip_subresources enqueue_delete_todos(member) + enqueue_unassign_issuables(member) if unassign_issuables after_execute(member: member) @@ -27,6 +28,12 @@ module Members private + def authorized?(member, destroy_bot) + return can_destroy_bot_member?(member) if destroy_bot + + can_destroy_member?(member) + end + def delete_subresources(member) return unless member.is_a?(GroupMember) && member.user && member.group @@ -54,6 +61,10 @@ module Members can?(current_user, destroy_member_permission(member), member) end + def can_destroy_bot_member?(member) + can?(current_user, destroy_bot_member_permission(member), member) + end + def destroy_member_permission(member) case member when GroupMember @@ -64,6 +75,20 @@ module Members raise "Unknown member type: #{member}!" end end + + def destroy_bot_member_permission(member) + raise "Unsupported bot member type: #{member}" unless member.is_a?(ProjectMember) + + :destroy_project_bot_member + end + + def enqueue_unassign_issuables(member) + source_type = member.is_a?(GroupMember) ? 'Group' : 'Project' + + member.run_after_commit_or_now do + MembersDestroyer::UnassignIssuablesWorker.perform_async(member.user_id, member.source_id, source_type) + end + end end end diff --git a/app/services/members/unassign_issuables_service.rb b/app/services/members/unassign_issuables_service.rb new file mode 100644 index 00000000000..95e07deb761 --- /dev/null +++ b/app/services/members/unassign_issuables_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Members + class UnassignIssuablesService + attr_reader :user, :entity + + def initialize(user, entity) + @user = user + @entity = entity + end + + def execute + return unless entity && user + + project_ids = entity.is_a?(Group) ? entity.all_projects.select(:id) : [entity.id] + + user.issue_assignees.on_issues(Issue.in_projects(project_ids).select(:id)).delete_all + user.merge_request_assignees.in_projects(project_ids).delete_all + + user.invalidate_cache_counts + end + end +end diff --git a/app/services/merge_requests/approval_service.rb b/app/services/merge_requests/approval_service.rb new file mode 100644 index 00000000000..150ec85fca9 --- /dev/null +++ b/app/services/merge_requests/approval_service.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module MergeRequests + class ApprovalService < MergeRequests::BaseService + def execute(merge_request) + return unless can_be_approved?(merge_request) + + approval = merge_request.approvals.new(user: current_user) + + return success unless save_approval(approval) + + reset_approvals_cache(merge_request) + create_event(merge_request) + create_approval_note(merge_request) + mark_pending_todos_as_done(merge_request) + execute_approval_hooks(merge_request, current_user) + + success + end + + private + + def can_be_approved?(merge_request) + current_user.can?(:approve_merge_request, merge_request) + end + + def reset_approvals_cache(merge_request) + merge_request.approvals.reset + end + + def execute_approval_hooks(merge_request, current_user) + # Only one approval is required for a merge request to be approved + execute_hooks(merge_request, 'approved') + end + + def save_approval(approval) + Approval.safe_ensure_unique do + approval.save + end + end + + def create_approval_note(merge_request) + SystemNoteService.approve_mr(merge_request, current_user) + end + + def mark_pending_todos_as_done(merge_request) + todo_service.resolve_todos_for_target(merge_request, current_user) + end + + def create_event(merge_request) + event_service.approve_mr(merge_request, current_user) + end + end +end + +MergeRequests::ApprovalService.prepend_if_ee('EE::MergeRequests::ApprovalService') diff --git a/app/services/merge_requests/base_service.rb b/app/services/merge_requests/base_service.rb index 7f7bfa29af7..7e301f311e9 100644 --- a/app/services/merge_requests/base_service.rb +++ b/app/services/merge_requests/base_service.rb @@ -2,6 +2,7 @@ module MergeRequests class BaseService < ::IssuableBaseService + extend ::Gitlab::Utils::Override include MergeRequests::AssignsMergeParams def create_note(merge_request, state = merge_request.state) @@ -29,6 +30,11 @@ module MergeRequests .execute_for_merge_request(merge_request) end + def cancel_review_app_jobs!(merge_request) + environments = merge_request.environments.in_review_folder.available + environments.each { |environment| environment.cancel_deployment_jobs! } + end + def source_project @source_project ||= merge_request.source_project end @@ -58,6 +64,12 @@ module MergeRequests super end + override :handle_quick_actions + def handle_quick_actions(merge_request) + super + handle_wip_event(merge_request) + end + def handle_wip_event(merge_request) if wip_event = params.delete(:wip_event) # We update the title that is provided in the params or we use the mr title @@ -90,10 +102,6 @@ module MergeRequests MergeRequests::CreatePipelineService.new(project, user).execute(merge_request) end - def can_use_merge_request_ref?(merge_request) - !merge_request.for_fork? - end - def abort_auto_merge(merge_request, reason) AutoMergeService.new(project, current_user).abort(merge_request, reason) end diff --git a/app/services/merge_requests/create_pipeline_service.rb b/app/services/merge_requests/create_pipeline_service.rb index f802aa44487..f9352f10fea 100644 --- a/app/services/merge_requests/create_pipeline_service.rb +++ b/app/services/merge_requests/create_pipeline_service.rb @@ -9,7 +9,7 @@ module MergeRequests end def create_detached_merge_request_pipeline(merge_request) - Ci::CreatePipelineService.new(merge_request.source_project, + Ci::CreatePipelineService.new(pipeline_project(merge_request), current_user, ref: pipeline_ref_for_detached_merge_request_pipeline(merge_request)) .execute(:merge_request_event, merge_request: merge_request) @@ -31,13 +31,29 @@ module MergeRequests private + def pipeline_project(merge_request) + if can_create_pipeline_in_target_project?(merge_request) + merge_request.target_project + else + merge_request.source_project + end + end + def pipeline_ref_for_detached_merge_request_pipeline(merge_request) - if can_use_merge_request_ref?(merge_request) + if can_create_pipeline_in_target_project?(merge_request) merge_request.ref_path else merge_request.source_branch end end + + def can_create_pipeline_in_target_project?(merge_request) + if Gitlab::Ci::Features.allow_to_create_merge_request_pipelines_in_target_project?(merge_request.target_project) + can?(current_user, :create_pipeline, merge_request.target_project) + else + merge_request.for_same_project? + end + end end end diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 1cdfba41432..ac84a13f437 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -33,12 +33,6 @@ module MergeRequests super end - # Override from IssuableBaseService - def handle_quick_actions_on_create(merge_request) - super - handle_wip_event(merge_request) - end - private def set_projects! diff --git a/app/services/merge_requests/ff_merge_service.rb b/app/services/merge_requests/ff_merge_service.rb index 6f1fa607ef9..b3896d61a78 100644 --- a/app/services/merge_requests/ff_merge_service.rb +++ b/app/services/merge_requests/ff_merge_service.rb @@ -16,7 +16,7 @@ module MergeRequests merge_request.target_branch, merge_request: merge_request) - if merge_request.squash + if merge_request.squash_on_merge? merge_request.update_column(:squash_commit_sha, merge_request.in_progress_merge_commit_sha) end diff --git a/app/services/merge_requests/merge_base_service.rb b/app/services/merge_requests/merge_base_service.rb index 27b5e31faab..fe09c92aab9 100644 --- a/app/services/merge_requests/merge_base_service.rb +++ b/app/services/merge_requests/merge_base_service.rb @@ -20,7 +20,7 @@ module MergeRequests def source strong_memoize(:source) do - if merge_request.squash + if merge_request.squash_on_merge? squash_sha! else merge_request.diff_head_sha diff --git a/app/services/merge_requests/merge_service.rb b/app/services/merge_requests/merge_service.rb index 8d57a76f7d0..961a7cb1ef6 100644 --- a/app/services/merge_requests/merge_service.rb +++ b/app/services/merge_requests/merge_service.rb @@ -27,6 +27,7 @@ module MergeRequests success end end + log_info("Merge process finished on JID #{merge_jid} with state #{state}") rescue MergeError => e handle_merge_error(log_message: e.message, save_message_on_model: true) @@ -56,6 +57,8 @@ module MergeRequests 'Only fast-forward merge is allowed for your project. Please update your source branch' elsif !@merge_request.mergeable? 'Merge request is not mergeable' + elsif !@merge_request.squash && project.squash_always? + 'This project requires squashing commits when merge requests are accepted.' end raise_error(error) if error diff --git a/app/services/merge_requests/post_merge_service.rb b/app/services/merge_requests/post_merge_service.rb index 0364c0dd479..fdf8f442297 100644 --- a/app/services/merge_requests/post_merge_service.rb +++ b/app/services/merge_requests/post_merge_service.rb @@ -18,6 +18,7 @@ module MergeRequests invalidate_cache_counts(merge_request, users: merge_request.assignees) merge_request.update_project_counter_caches delete_non_latest_diffs(merge_request) + cancel_review_app_jobs!(merge_request) cleanup_environments(merge_request) end diff --git a/app/services/merge_requests/remove_approval_service.rb b/app/services/merge_requests/remove_approval_service.rb new file mode 100644 index 00000000000..3164d0b4069 --- /dev/null +++ b/app/services/merge_requests/remove_approval_service.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module MergeRequests + class RemoveApprovalService < MergeRequests::BaseService + # rubocop: disable CodeReuse/ActiveRecord + def execute(merge_request) + return unless merge_request.approved_by?(current_user) + + # paranoid protection against running wrong deletes + return unless merge_request.id && current_user.id + + approval = merge_request.approvals.where(user: current_user) + + trigger_approval_hooks(merge_request) do + next unless approval.destroy_all # rubocop: disable Cop/DestroyAll + + reset_approvals_cache(merge_request) + create_note(merge_request) + end + + success + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + def reset_approvals_cache(merge_request) + merge_request.approvals.reset + end + + def trigger_approval_hooks(merge_request) + yield + + execute_hooks(merge_request, 'unapproved') + end + + def create_note(merge_request) + SystemNoteService.unapprove_mr(merge_request, current_user) + end + end +end + +MergeRequests::RemoveApprovalService.prepend_if_ee('EE::MergeRequests::RemoveApprovalService') diff --git a/app/services/merge_requests/squash_service.rb b/app/services/merge_requests/squash_service.rb index 4b04d42b48e..faa2e921581 100644 --- a/app/services/merge_requests/squash_service.rb +++ b/app/services/merge_requests/squash_service.rb @@ -11,11 +11,14 @@ module MergeRequests return success(squash_sha: merge_request.diff_head_sha) end + return error(s_('MergeRequests|This project does not allow squashing commits when merge requests are accepted.')) if squash_forbidden? + if squash_in_progress? return error(s_('MergeRequests|Squash task canceled: another squash is already in progress.')) end squash! || error(s_('MergeRequests|Failed to squash. Should be done manually.')) + rescue SquashInProgressError error(s_('MergeRequests|An error occurred while checking whether another squash is in progress.')) end @@ -40,6 +43,10 @@ module MergeRequests raise SquashInProgressError, e.message end + def squash_forbidden? + target_project.squash_never? + end + def repository target_project.repository end diff --git a/app/services/merge_requests/update_service.rb b/app/services/merge_requests/update_service.rb index 561695baeab..29e0c22b155 100644 --- a/app/services/merge_requests/update_service.rb +++ b/app/services/merge_requests/update_service.rb @@ -2,6 +2,8 @@ module MergeRequests class UpdateService < MergeRequests::BaseService + extend ::Gitlab::Utils::Override + def execute(merge_request) # We don't allow change of source/target projects and source branch # after merge request was created @@ -9,14 +11,11 @@ module MergeRequests params.delete(:target_project_id) params.delete(:source_branch) - merge_from_quick_action(merge_request) if params[:merge] - if merge_request.closed_without_fork? params.delete(:target_branch) params.delete(:force_remove_source_branch) end - handle_wip_event(merge_request) update_task_event(merge_request) || update(merge_request) end @@ -77,26 +76,6 @@ module MergeRequests todo_service.update_merge_request(merge_request, current_user) end - def merge_from_quick_action(merge_request) - last_diff_sha = params.delete(:merge) - - if Feature.enabled?(:merge_orchestration_service, merge_request.project, default_enabled: true) - MergeRequests::MergeOrchestrationService - .new(project, current_user, { sha: last_diff_sha }) - .execute(merge_request) - else - return unless merge_request.mergeable_with_quick_action?(current_user, last_diff_sha: last_diff_sha) - - merge_request.update(merge_error: nil) - - if merge_request.head_pipeline_active? - AutoMergeService.new(project, current_user, { sha: last_diff_sha }).execute(merge_request, AutoMergeService::STRATEGY_MERGE_WHEN_PIPELINE_SUCCEEDS) - else - merge_request.merge_async(current_user.id, { sha: last_diff_sha }) - end - end - end - def reopen_service MergeRequests::ReopenService end @@ -134,6 +113,37 @@ module MergeRequests issuable, issuable.project, current_user, branch_type, old_branch, new_branch) end + + override :handle_quick_actions + def handle_quick_actions(merge_request) + super + merge_from_quick_action(merge_request) if params[:merge] + end + + def merge_from_quick_action(merge_request) + last_diff_sha = params.delete(:merge) + + if Feature.enabled?(:merge_orchestration_service, merge_request.project, default_enabled: true) + MergeRequests::MergeOrchestrationService + .new(project, current_user, { sha: last_diff_sha }) + .execute(merge_request) + else + return unless merge_request.mergeable_with_quick_action?(current_user, last_diff_sha: last_diff_sha) + + merge_request.update(merge_error: nil) + + if merge_request.head_pipeline_active? + AutoMergeService.new(project, current_user, { sha: last_diff_sha }).execute(merge_request, AutoMergeService::STRATEGY_MERGE_WHEN_PIPELINE_SUCCEEDS) + else + merge_request.merge_async(current_user.id, { sha: last_diff_sha }) + end + end + end + + override :quick_action_options + def quick_action_options + { merge_request_diff_head_sha: params.delete(:merge_request_diff_head_sha) } + end end end diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb index c2a0f22e73e..5fa127d64b2 100644 --- a/app/services/metrics/dashboard/base_service.rb +++ b/app/services/metrics/dashboard/base_service.rb @@ -10,7 +10,8 @@ module Metrics STAGES = ::Gitlab::Metrics::Dashboard::Stages SEQUENCE = [ STAGES::CommonMetricsInserter, - STAGES::EndpointInserter, + STAGES::MetricEndpointInserter, + STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter, STAGES::Sorter, STAGES::AlertsInserter, @@ -36,6 +37,14 @@ module Metrics Gitlab::Metrics::Dashboard::Cache.fetch(cache_key) { get_raw_dashboard } end + # Should return true if this dashboard service is for an out-of-the-box + # dashboard. + # This method is overridden in app/services/metrics/dashboard/predefined_dashboard_service.rb. + # @return Boolean + def self.out_of_the_box_dashboard? + false + end + private # Determines whether users should be able to view @@ -83,6 +92,17 @@ module Metrics params[:dashboard_path] end + def load_yaml(data) + ::Gitlab::Config::Loader::Yaml.new(data).load_raw! + rescue Gitlab::Config::Loader::Yaml::NotHashError + # Raise more informative error in app/models/performance_monitoring/prometheus_dashboard.rb. + {} + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => exception + raise Gitlab::Metrics::Dashboard::Errors::LayoutError, exception.message + rescue Gitlab::Config::Loader::FormatError + raise Gitlab::Metrics::Dashboard::Errors::LayoutError, _('Invalid yaml') + end + # @return [Hash] an unmodified dashboard def get_raw_dashboard raise NotImplementedError diff --git a/app/services/metrics/dashboard/clone_dashboard_service.rb b/app/services/metrics/dashboard/clone_dashboard_service.rb index 3ca25b3bd9b..a6bece391f2 100644 --- a/app/services/metrics/dashboard/clone_dashboard_service.rb +++ b/app/services/metrics/dashboard/clone_dashboard_service.rb @@ -6,30 +6,33 @@ module Metrics module Dashboard class CloneDashboardService < ::BaseService include Stepable + include Gitlab::Utils::StrongMemoize ALLOWED_FILE_TYPE = '.yml' USER_DASHBOARDS_DIR = ::Metrics::Dashboard::CustomDashboardService::DASHBOARD_ROOT + SEQUENCES = { + ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH => [ + ::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, + ::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter, + ::Gitlab::Metrics::Dashboard::Stages::Sorter + ].freeze, + + ::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH => [ + ::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter + ].freeze, + + ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH => [ + ::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, + ::Gitlab::Metrics::Dashboard::Stages::Sorter + ].freeze + }.freeze steps :check_push_authorized, - :check_branch_name, - :check_file_type, - :check_dashboard_template, - :create_file, - :refresh_repository_method_caches - - class << self - def allowed_dashboard_templates - @allowed_dashboard_templates ||= Set[::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH].freeze - end - - def sequences - @sequences ||= { - ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH => [::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, - ::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter, - ::Gitlab::Metrics::Dashboard::Stages::Sorter].freeze - }.freeze - end - end + :check_branch_name, + :check_file_type, + :check_dashboard_template, + :create_file, + :refresh_repository_method_caches def execute execute_steps @@ -56,8 +59,12 @@ module Metrics success(result) end + # Only allow out of the box metrics dashboards to be cloned. This can be + # changed to allow cloning of any metrics dashboard, if desired. + # However, only metrics dashboards should be allowed. If any file is + # allowed to be cloned, this will become a security risk. def check_dashboard_template(result) - return error(_('Not found.'), :not_found) unless self.class.allowed_dashboard_templates.include?(params[:dashboard]) + return error(_('Not found.'), :not_found) unless dashboard_service&.out_of_the_box_dashboard? success(result) end @@ -78,6 +85,12 @@ module Metrics success(result.merge(http_status: :created, dashboard: dashboard_details)) end + def dashboard_service + strong_memoize(:dashboard_service) do + Gitlab::Metrics::Dashboard::ServiceSelector.call(dashboard_service_options) + end + end + def dashboard_attrs { commit_message: params[:commit_message], @@ -149,14 +162,19 @@ module Metrics end def raw_dashboard - YAML.safe_load(File.read(Rails.root.join(dashboard_template))) + dashboard_service.new(project, current_user, dashboard_service_options).raw_dashboard + end + + def dashboard_service_options + { + embedded: false, + dashboard_path: dashboard_template + } end def sequence - self.class.sequences[dashboard_template] + SEQUENCES[dashboard_template] || [] end end end end - -Metrics::Dashboard::CloneDashboardService.prepend_if_ee('EE::Metrics::Dashboard::CloneDashboardService') diff --git a/app/services/metrics/dashboard/cluster_dashboard_service.rb b/app/services/metrics/dashboard/cluster_dashboard_service.rb new file mode 100644 index 00000000000..bfd5abf1126 --- /dev/null +++ b/app/services/metrics/dashboard/cluster_dashboard_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Fetches the system metrics dashboard and formats the output. +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class ClusterDashboardService < ::Metrics::Dashboard::PredefinedDashboardService + DASHBOARD_PATH = 'config/prometheus/cluster_metrics.yml' + DASHBOARD_NAME = 'Cluster' + + # SHA256 hash of dashboard content + DASHBOARD_VERSION = '9349afc1d96329c08ab478ea0b77db94ee5cc2549b8c754fba67a7f424666b22' + + SEQUENCE = [ + STAGES::ClusterEndpointInserter, + STAGES::PanelIdsInserter, + STAGES::Sorter + ].freeze + + class << self + def valid_params?(params) + # support selecting this service by cluster id via .find + # Use super to support selecting this service by dashboard_path via .find_raw + (params[:cluster].present? && params[:embedded] != 'true') || super + end + end + + # Permissions are handled at the controller level + def allowed? + true + end + + private + + def dashboard_version + DASHBOARD_VERSION + end + end + end +end diff --git a/app/services/metrics/dashboard/cluster_metrics_embed_service.rb b/app/services/metrics/dashboard/cluster_metrics_embed_service.rb new file mode 100644 index 00000000000..6fb39ed3004 --- /dev/null +++ b/app/services/metrics/dashboard/cluster_metrics_embed_service.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true +# +module Metrics + module Dashboard + class ClusterMetricsEmbedService < Metrics::Dashboard::DynamicEmbedService + class << self + def valid_params?(params) + [ + params[:cluster], + embedded?(params[:embedded]), + params[:group].present?, + params[:title].present?, + params[:y_label].present? + ].all? + end + end + + private + + # Permissions are handled at the controller level + def allowed? + true + end + + def dashboard_path + ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH + end + + def sequence + [ + STAGES::ClusterEndpointInserter, + STAGES::PanelIdsInserter + ] + end + end + end +end diff --git a/app/services/metrics/dashboard/custom_dashboard_service.rb b/app/services/metrics/dashboard/custom_dashboard_service.rb index 77173813a4f..741738cc3af 100644 --- a/app/services/metrics/dashboard/custom_dashboard_service.rb +++ b/app/services/metrics/dashboard/custom_dashboard_service.rb @@ -21,7 +21,8 @@ module Metrics path: filepath, display_name: name_for_path(filepath), default: false, - system_dashboard: false + system_dashboard: false, + out_of_the_box_dashboard: out_of_the_box_dashboard? } end end @@ -42,7 +43,7 @@ module Metrics def get_raw_dashboard yml = self.class.file_finder(project).read(dashboard_path) - YAML.safe_load(yml) + load_yaml(yml) end def cache_key diff --git a/app/services/metrics/dashboard/gitlab_alert_embed_service.rb b/app/services/metrics/dashboard/gitlab_alert_embed_service.rb index 38e89d392ad..08d65413e1d 100644 --- a/app/services/metrics/dashboard/gitlab_alert_embed_service.rb +++ b/app/services/metrics/dashboard/gitlab_alert_embed_service.rb @@ -11,7 +11,7 @@ module Metrics include Gitlab::Utils::StrongMemoize SEQUENCE = [ - STAGES::EndpointInserter, + STAGES::MetricEndpointInserter, STAGES::PanelIdsInserter ].freeze diff --git a/app/services/metrics/dashboard/grafana_metric_embed_service.rb b/app/services/metrics/dashboard/grafana_metric_embed_service.rb index d9ce2c5e905..8e72a185406 100644 --- a/app/services/metrics/dashboard/grafana_metric_embed_service.rb +++ b/app/services/metrics/dashboard/grafana_metric_embed_service.rb @@ -80,7 +80,7 @@ module Metrics def fetch_dashboard uid = GrafanaUidParser.new(grafana_url, project).parse - raise DashboardProcessingError.new('Dashboard uid not found') unless uid + raise DashboardProcessingError.new(_('Dashboard uid not found')) unless uid response = client.get_dashboard(uid: uid) @@ -89,7 +89,7 @@ module Metrics def fetch_datasource(dashboard) name = DatasourceNameParser.new(grafana_url, dashboard).parse - raise DashboardProcessingError.new('Datasource name not found') unless name + raise DashboardProcessingError.new(_('Datasource name not found')) unless name response = client.get_datasource(name: name) @@ -115,7 +115,7 @@ module Metrics def parse_json(json) Gitlab::Json.parse(json, symbolize_names: true) rescue JSON::ParserError - raise DashboardProcessingError.new('Grafana response contains invalid json') + raise DashboardProcessingError.new(_('Grafana response contains invalid json')) end end diff --git a/app/services/metrics/dashboard/pod_dashboard_service.rb b/app/services/metrics/dashboard/pod_dashboard_service.rb index 16b87d2d587..8699189deac 100644 --- a/app/services/metrics/dashboard/pod_dashboard_service.rb +++ b/app/services/metrics/dashboard/pod_dashboard_service.rb @@ -5,6 +5,15 @@ module Metrics class PodDashboardService < ::Metrics::Dashboard::PredefinedDashboardService DASHBOARD_PATH = 'config/prometheus/pod_metrics.yml' DASHBOARD_NAME = 'Pod Health' + + # SHA256 hash of dashboard content + DASHBOARD_VERSION = 'f12f641d2575d5dcb69e2c633ff5231dbd879ad35020567d8fc4e1090bfdb4b4' + + private + + def dashboard_version + DASHBOARD_VERSION + end end end end diff --git a/app/services/metrics/dashboard/predefined_dashboard_service.rb b/app/services/metrics/dashboard/predefined_dashboard_service.rb index f454df63773..c21083475f0 100644 --- a/app/services/metrics/dashboard/predefined_dashboard_service.rb +++ b/app/services/metrics/dashboard/predefined_dashboard_service.rb @@ -10,7 +10,8 @@ module Metrics DASHBOARD_NAME = nil SEQUENCE = [ - STAGES::EndpointInserter, + STAGES::MetricEndpointInserter, + STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter, STAGES::Sorter ].freeze @@ -23,12 +24,20 @@ module Metrics def matching_dashboard?(filepath) filepath == self::DASHBOARD_PATH end + + def out_of_the_box_dashboard? + true + end end private + def dashboard_version + raise NotImplementedError + end + def cache_key - "metrics_dashboard_#{dashboard_path}" + "metrics_dashboard_#{dashboard_path}_#{dashboard_version}" end def dashboard_path @@ -39,7 +48,7 @@ module Metrics def get_raw_dashboard yml = File.read(Rails.root.join(dashboard_path)) - YAML.safe_load(yml) + load_yaml(yml) end def sequence diff --git a/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb b/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb index 8599c23c206..f1f5cd7d77e 100644 --- a/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb +++ b/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb @@ -8,9 +8,13 @@ module Metrics DASHBOARD_PATH = 'config/prometheus/self_monitoring_default.yml' DASHBOARD_NAME = N_('Default dashboard') + # SHA256 hash of dashboard content + DASHBOARD_VERSION = '1dff3e3cb76e73c8e368823c98b34c61aec0d141978450dea195a3b3dc2415d6' + SEQUENCE = [ STAGES::CustomMetricsInserter, - STAGES::EndpointInserter, + STAGES::MetricEndpointInserter, + STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter, STAGES::Sorter ].freeze @@ -25,7 +29,8 @@ module Metrics path: DASHBOARD_PATH, display_name: _(DASHBOARD_NAME), default: true, - system_dashboard: false + system_dashboard: false, + out_of_the_box_dashboard: out_of_the_box_dashboard? }] end @@ -33,6 +38,12 @@ module Metrics params[:dashboard_path].nil? && params[:environment]&.project&.self_monitoring? end end + + private + + def dashboard_version + DASHBOARD_VERSION + end end end end diff --git a/app/services/metrics/dashboard/system_dashboard_service.rb b/app/services/metrics/dashboard/system_dashboard_service.rb index db5599b4def..5c3562b8ca0 100644 --- a/app/services/metrics/dashboard/system_dashboard_service.rb +++ b/app/services/metrics/dashboard/system_dashboard_service.rb @@ -8,11 +8,15 @@ module Metrics DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' DASHBOARD_NAME = N_('Default dashboard') + # SHA256 hash of dashboard content + DASHBOARD_VERSION = '4685fe386c25b1a786b3be18f79bb2ee9828019003e003816284cdb634fa3e13' + SEQUENCE = [ STAGES::CommonMetricsInserter, STAGES::CustomMetricsInserter, STAGES::CustomMetricsDetailsInserter, - STAGES::EndpointInserter, + STAGES::MetricEndpointInserter, + STAGES::VariableEndpointInserter, STAGES::PanelIdsInserter, STAGES::Sorter, STAGES::AlertsInserter @@ -24,10 +28,17 @@ module Metrics path: DASHBOARD_PATH, display_name: _(DASHBOARD_NAME), default: true, - system_dashboard: true + system_dashboard: true, + out_of_the_box_dashboard: out_of_the_box_dashboard? }] end end + + private + + def dashboard_version + DASHBOARD_VERSION + end end end end diff --git a/app/services/metrics/dashboard/transient_embed_service.rb b/app/services/metrics/dashboard/transient_embed_service.rb index cb6ca215447..0a9c4bc7b86 100644 --- a/app/services/metrics/dashboard/transient_embed_service.rb +++ b/app/services/metrics/dashboard/transient_embed_service.rb @@ -30,7 +30,7 @@ module Metrics override :sequence def sequence - [STAGES::EndpointInserter] + [STAGES::MetricEndpointInserter] end override :identifiers @@ -39,7 +39,7 @@ module Metrics end def invalid_embed_json!(message) - raise DashboardProcessingError.new("Parsing error for param :embed_json. #{message}") + raise DashboardProcessingError.new(_("Parsing error for param :embed_json. %{message}") % { message: message }) end end end diff --git a/app/services/namespaces/check_storage_size_service.rb b/app/services/namespaces/check_storage_size_service.rb deleted file mode 100644 index 57d2645a0c8..00000000000 --- a/app/services/namespaces/check_storage_size_service.rb +++ /dev/null @@ -1,95 +0,0 @@ -# frozen_string_literal: true - -module Namespaces - class CheckStorageSizeService - include ActiveSupport::NumberHelper - include Gitlab::Allowable - include Gitlab::Utils::StrongMemoize - - def initialize(namespace, user) - @root_namespace = namespace.root_ancestor - @root_storage_size = Namespace::RootStorageSize.new(root_namespace) - @user = user - end - - def execute - return ServiceResponse.success unless Feature.enabled?(:namespace_storage_limit, root_namespace) - return ServiceResponse.success if alert_level == :none - - if root_storage_size.above_size_limit? - ServiceResponse.error(message: above_size_limit_message, payload: payload) - else - ServiceResponse.success(payload: payload) - end - end - - private - - attr_reader :root_namespace, :root_storage_size, :user - - USAGE_THRESHOLDS = { - none: 0.0, - info: 0.5, - warning: 0.75, - alert: 0.95, - error: 1.0 - }.freeze - - def payload - return {} unless can?(user, :admin_namespace, root_namespace) - - { - explanation_message: explanation_message, - usage_message: usage_message, - alert_level: alert_level, - root_namespace: root_namespace - } - end - - def explanation_message - root_storage_size.above_size_limit? ? above_size_limit_message : below_size_limit_message - end - - def usage_message - s_("You reached %{usage_in_percent} of %{namespace_name}'s storage capacity (%{used_storage} of %{storage_limit})" % current_usage_params) - end - - def alert_level - strong_memoize(:alert_level) do - usage_ratio = root_storage_size.usage_ratio - current_level = USAGE_THRESHOLDS.each_key.first - - USAGE_THRESHOLDS.each do |level, threshold| - current_level = level if usage_ratio >= threshold - end - - current_level - end - end - - def below_size_limit_message - s_("If you reach 100%% storage capacity, you will not be able to: %{base_message}" % { base_message: base_message } ) - end - - def above_size_limit_message - s_("%{namespace_name} is now read-only. You cannot: %{base_message}" % { namespace_name: root_namespace.name, base_message: base_message }) - end - - def base_message - s_("push to your repository, create pipelines, create issues or add comments. To reduce storage capacity, delete unused repositories, artifacts, wikis, issues, and pipelines.") - end - - def current_usage_params - { - usage_in_percent: number_to_percentage(root_storage_size.usage_ratio * 100, precision: 0), - namespace_name: root_namespace.name, - used_storage: formatted(root_storage_size.current_size), - storage_limit: formatted(root_storage_size.limit) - } - end - - def formatted(number) - number_to_human_size(number, delimiter: ',', precision: 2) - end - end -end diff --git a/app/services/notes/post_process_service.rb b/app/services/notes/post_process_service.rb index 0e455c641ce..4f3b2000e9a 100644 --- a/app/services/notes/post_process_service.rb +++ b/app/services/notes/post_process_service.rb @@ -10,13 +10,13 @@ module Notes def execute # Skip system notes, like status changes and cross-references and awards - unless @note.system? - EventCreateService.new.leave_note(@note, @note.author) + unless note.system? + EventCreateService.new.leave_note(note, note.author) - return if @note.for_personal_snippet? + return if note.for_personal_snippet? - @note.create_cross_references! - ::SystemNoteService.design_discussion_added(@note) if create_design_discussion_system_note? + note.create_cross_references! + ::SystemNoteService.design_discussion_added(note) if create_design_discussion_system_note? execute_note_hooks end @@ -25,21 +25,21 @@ module Notes private def create_design_discussion_system_note? - @note && @note.for_design? && @note.start_of_discussion? + note && note.for_design? && note.start_of_discussion? end def hook_data - Gitlab::DataBuilder::Note.build(@note, @note.author) + Gitlab::DataBuilder::Note.build(note, note.author) end def execute_note_hooks - return unless @note.project + return unless note.project note_data = hook_data - hooks_scope = @note.confidential?(include_noteable: true) ? :confidential_note_hooks : :note_hooks + hooks_scope = note.confidential?(include_noteable: true) ? :confidential_note_hooks : :note_hooks - @note.project.execute_hooks(note_data, hooks_scope) - @note.project.execute_services(note_data, hooks_scope) + note.project.execute_hooks(note_data, hooks_scope) + note.project.execute_services(note_data, hooks_scope) end end end diff --git a/app/services/notes/quick_actions_service.rb b/app/services/notes/quick_actions_service.rb index 7e6568b5b25..c670f01e502 100644 --- a/app/services/notes/quick_actions_service.rb +++ b/app/services/notes/quick_actions_service.rb @@ -41,7 +41,7 @@ module Notes @interpret_service = QuickActions::InterpretService.new(project, current_user, options) - @interpret_service.execute(note.note, note.noteable) + interpret_service.execute(note.note, note.noteable) end # Applies updates extracted to note#noteable diff --git a/app/services/notes/update_service.rb b/app/services/notes/update_service.rb index 444656348ed..047848fd1a3 100644 --- a/app/services/notes/update_service.rb +++ b/app/services/notes/update_service.rb @@ -10,6 +10,7 @@ module Notes note.assign_attributes(params.merge(updated_by: current_user)) note.with_transaction_returning_status do + update_confidentiality(note) note.save end @@ -79,6 +80,15 @@ module Notes TodoService.new.update_note(note, current_user, old_mentioned_users) end + + # This method updates confidentiality of all discussion notes at once + def update_confidentiality(note) + return unless params.key?(:confidential) + return unless note.is_a?(DiscussionNote) # we don't need to do bulk update for single notes + return unless note.start_of_discussion? # don't update all notes if a response is being updated + + Note.id_in(note.discussion.notes.map(&:id)).update_all(confidential: params[:confidential]) + end end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 73e60ac8420..a4e935a8cf5 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -294,6 +294,7 @@ class NotificationService return true if note.system_note_with_references? send_new_note_notifications(note) + send_service_desk_notification(note) end def send_new_note_notifications(note) @@ -305,6 +306,21 @@ class NotificationService end end + def send_service_desk_notification(note) + return unless Gitlab::ServiceDesk.supported? + return unless note.noteable_type == 'Issue' + + issue = note.noteable + support_bot = User.support_bot + + return unless issue.service_desk_reply_to.present? + return unless issue.project.service_desk_enabled? + return if note.author == support_bot + return unless issue.subscribed?(support_bot, issue.project) + + mailer.service_desk_new_note_email(issue.id, note.id).deliver_later + end + # Notify users when a new release is created def send_new_release_notifications(release) recipients = NotificationRecipients::BuildService.build_new_release_recipients(release) @@ -566,6 +582,14 @@ class NotificationService end end + def merge_when_pipeline_succeeds(merge_request, current_user) + recipients = ::NotificationRecipients::BuildService.build_recipients(merge_request, current_user, action: 'merge_when_pipeline_succeeds') + + recipients.each do |recipient| + mailer.merge_when_pipeline_succeeds_email(recipient.user.id, merge_request.id, current_user.id).deliver_later + end + end + protected def new_resource_email(target, method) diff --git a/app/services/packages/composer/composer_json_service.rb b/app/services/packages/composer/composer_json_service.rb new file mode 100644 index 00000000000..6ffb5a77da3 --- /dev/null +++ b/app/services/packages/composer/composer_json_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Packages + module Composer + class ComposerJsonService + def initialize(project, target) + @project, @target = project, target + end + + def execute + composer_json + end + + private + + def composer_json + composer_file = @project.repository.blob_at(@target, 'composer.json') + + composer_file_not_found! unless composer_file + + Gitlab::Json.parse(composer_file.data) + rescue JSON::ParserError + raise 'Could not parse composer.json file. Invalid JSON.' + end + + def composer_file_not_found! + raise 'The file composer.json was not found.' + end + end + end +end diff --git a/app/services/packages/composer/create_package_service.rb b/app/services/packages/composer/create_package_service.rb new file mode 100644 index 00000000000..ad5d267698b --- /dev/null +++ b/app/services/packages/composer/create_package_service.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Packages + module Composer + class CreatePackageService < BaseService + include ::Gitlab::Utils::StrongMemoize + + def execute + # fetches json outside of transaction + composer_json + + ::Packages::Package.transaction do + ::Packages::Composer::Metadatum.upsert( + package_id: created_package.id, + target_sha: target, + composer_json: composer_json + ) + end + end + + private + + def created_package + project + .packages + .composer + .safe_find_or_create_by!(name: package_name, version: package_version) + end + + def composer_json + strong_memoize(:composer_json) do + ::Packages::Composer::ComposerJsonService.new(project, target).execute + end + end + + def package_name + composer_json['name'] + end + + def target + (branch || tag).target + end + + def branch + params[:branch] + end + + def tag + params[:tag] + end + + def package_version + ::Packages::Composer::VersionParserService.new(tag_name: tag&.name, branch_name: branch&.name).execute + end + end + end +end diff --git a/app/services/packages/composer/version_parser_service.rb b/app/services/packages/composer/version_parser_service.rb new file mode 100644 index 00000000000..76dfd7a14bd --- /dev/null +++ b/app/services/packages/composer/version_parser_service.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Packages + module Composer + class VersionParserService + def initialize(tag_name: nil, branch_name: nil) + @tag_name, @branch_name = tag_name, branch_name + end + + def execute + if @tag_name.present? + @tag_name.match(Gitlab::Regex.composer_package_version_regex).captures[0] + elsif @branch_name.present? + branch_sufix_or_prefix(@branch_name.match(Gitlab::Regex.composer_package_version_regex)) + end + end + + private + + def branch_sufix_or_prefix(match) + if match + if match.captures[1] == '.x' + match.captures[0] + '-dev' + else + match.captures[0] + '.x-dev' + end + else + "dev-#{@branch_name}" + end + end + end + end +end diff --git a/app/services/packages/conan/create_package_file_service.rb b/app/services/packages/conan/create_package_file_service.rb new file mode 100644 index 00000000000..2db5c4e507b --- /dev/null +++ b/app/services/packages/conan/create_package_file_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Packages + module Conan + class CreatePackageFileService + attr_reader :package, :file, :params + + def initialize(package, file, params) + @package = package + @file = file + @params = params + end + + def execute + package.package_files.create!( + file: file, + size: params['file.size'], + file_name: params[:file_name], + file_sha1: params['file.sha1'], + file_md5: params['file.md5'], + conan_file_metadatum_attributes: { + recipe_revision: params[:recipe_revision], + package_revision: params[:package_revision], + conan_package_reference: params[:conan_package_reference], + conan_file_type: params[:conan_file_type] + } + ) + end + end + end +end diff --git a/app/services/packages/conan/create_package_service.rb b/app/services/packages/conan/create_package_service.rb new file mode 100644 index 00000000000..22a0436c5fb --- /dev/null +++ b/app/services/packages/conan/create_package_service.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Packages + module Conan + class CreatePackageService < BaseService + def execute + project.packages.create!( + name: params[:package_name], + version: params[:package_version], + package_type: :conan, + conan_metadatum_attributes: { + package_username: params[:package_username], + package_channel: params[:package_channel] + } + ) + end + end + end +end diff --git a/app/services/packages/conan/search_service.rb b/app/services/packages/conan/search_service.rb new file mode 100644 index 00000000000..4513616bad2 --- /dev/null +++ b/app/services/packages/conan/search_service.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Packages + module Conan + class SearchService < BaseService + include ActiveRecord::Sanitization::ClassMethods + + WILDCARD = '*' + RECIPE_SEPARATOR = '@' + + def initialize(user, params) + super(nil, user, params) + end + + def execute + ServiceResponse.success(payload: { results: search_results }) + end + + private + + def search_results + return [] if wildcard_query? + + return search_for_single_package(sanitized_query) if params[:query].include?(RECIPE_SEPARATOR) + + search_packages(build_query) + end + + def wildcard_query? + params[:query] == WILDCARD + end + + def build_query + return "#{sanitized_query}%" if params[:query].end_with?(WILDCARD) + + sanitized_query + end + + def search_packages(query) + ::Packages::Conan::PackageFinder.new(current_user, query: query).execute.map(&:conan_recipe) + end + + def search_for_single_package(query) + name, version, username, _ = query.split(/[@\/]/) + full_path = Packages::Conan::Metadatum.full_path_from(package_username: username) + project = Project.find_by_full_path(full_path) + return unless current_user.can?(:read_package, project) + + result = project.packages.with_name(name).with_version(version).order_created.last + [result&.conan_recipe].compact + end + + def sanitized_query + @sanitized_query ||= sanitize_sql_like(params[:query].delete(WILDCARD)) + end + end + end +end diff --git a/app/services/packages/create_dependency_service.rb b/app/services/packages/create_dependency_service.rb new file mode 100644 index 00000000000..2999885d55d --- /dev/null +++ b/app/services/packages/create_dependency_service.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true +module Packages + class CreateDependencyService < BaseService + attr_reader :package, :dependencies + + def initialize(package, dependencies) + @package = package + @dependencies = dependencies + end + + def execute + Packages::DependencyLink.dependency_types.each_key do |type| + create_dependency(type) + end + end + + private + + def create_dependency(type) + return unless dependencies[type].is_a?(Hash) + + names_and_version_patterns = dependencies[type] + existing_ids, existing_names = find_existing_ids_and_names(names_and_version_patterns) + dependencies_to_insert = names_and_version_patterns + + if existing_names.any? + dependencies_to_insert = names_and_version_patterns.reject { |k, _| k.in?(existing_names) } + end + + ActiveRecord::Base.transaction do + inserted_ids = bulk_insert_package_dependencies(dependencies_to_insert) + bulk_insert_package_dependency_links(type, (existing_ids + inserted_ids)) + end + end + + def find_existing_ids_and_names(names_and_version_patterns) + ids_and_names = Packages::Dependency.for_package_names_and_version_patterns(names_and_version_patterns) + .pluck_ids_and_names + ids = ids_and_names.map(&:first) || [] + names = ids_and_names.map(&:second) || [] + [ids, names] + end + + def bulk_insert_package_dependencies(names_and_version_patterns) + return [] if names_and_version_patterns.empty? + + rows = names_and_version_patterns.map do |name, version_pattern| + { + name: name, + version_pattern: version_pattern + } + end + + ids = database.bulk_insert(Packages::Dependency.table_name, rows, return_ids: true, on_conflict: :do_nothing) + return ids if ids.size == names_and_version_patterns.size + + Packages::Dependency.uncached do + # The bulk_insert statement above do not dirty the query cache. To make + # sure that the results are fresh from the database and not from a stalled + # and potentially wrong cache, this query has to be done with the query + # chache disabled. + Packages::Dependency.ids_for_package_names_and_version_patterns(names_and_version_patterns) + end + end + + def bulk_insert_package_dependency_links(type, dependency_ids) + rows = dependency_ids.map do |dependency_id| + { + package_id: package.id, + dependency_id: dependency_id, + dependency_type: Packages::DependencyLink.dependency_types[type.to_s] + } + end + + database.bulk_insert(Packages::DependencyLink.table_name, rows) + end + + def database + ::Gitlab::Database + end + end +end diff --git a/app/services/packages/create_package_file_service.rb b/app/services/packages/create_package_file_service.rb new file mode 100644 index 00000000000..0ebceeee779 --- /dev/null +++ b/app/services/packages/create_package_file_service.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +module Packages + class CreatePackageFileService + attr_reader :package, :params + + def initialize(package, params) + @package = package + @params = params + end + + def execute + package.package_files.create!( + file: params[:file], + size: params[:size], + file_name: params[:file_name], + file_sha1: params[:file_sha1], + file_sha256: params[:file_sha256], + file_md5: params[:file_md5] + ) + end + end +end diff --git a/app/services/packages/maven/create_package_service.rb b/app/services/packages/maven/create_package_service.rb new file mode 100644 index 00000000000..aca5d28ca98 --- /dev/null +++ b/app/services/packages/maven/create_package_service.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +module Packages + module Maven + class CreatePackageService < BaseService + def execute + app_group, _, app_name = params[:name].rpartition('/') + app_group.tr!('/', '.') + + package = project.packages.create!( + name: params[:name], + version: params[:version], + package_type: :maven, + maven_metadatum_attributes: { + path: params[:path], + app_group: app_group, + app_name: app_name, + app_version: params[:version] + } + ) + + build = params[:build] + package.create_build_info!(pipeline: build.pipeline) if build.present? + + package + end + end + end +end diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb new file mode 100644 index 00000000000..50a008843ad --- /dev/null +++ b/app/services/packages/maven/find_or_create_package_service.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +module Packages + module Maven + class FindOrCreatePackageService < BaseService + MAVEN_METADATA_FILE = 'maven-metadata.xml'.freeze + + def execute + package = ::Packages::Maven::PackageFinder + .new(params[:path], current_user, project: project).execute + + unless package + if params[:file_name] == MAVEN_METADATA_FILE + # Maven uploads several files during `mvn deploy` in next order: + # - my-company/my-app/1.0-SNAPSHOT/my-app.jar + # - my-company/my-app/1.0-SNAPSHOT/my-app.pom + # - my-company/my-app/1.0-SNAPSHOT/maven-metadata.xml + # - my-company/my-app/maven-metadata.xml + # + # The last xml file does not have VERSION in URL because it contains + # information about all versions. + package_name, version = params[:path], nil + else + package_name, _, version = params[:path].rpartition('/') + end + + package_params = { + name: package_name, + path: params[:path], + version: version, + build: params[:build] + } + + package = ::Packages::Maven::CreatePackageService + .new(project, current_user, package_params).execute + end + + package + end + end + end +end diff --git a/app/services/packages/npm/create_package_service.rb b/app/services/packages/npm/create_package_service.rb new file mode 100644 index 00000000000..cf927683ce9 --- /dev/null +++ b/app/services/packages/npm/create_package_service.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +module Packages + module Npm + class CreatePackageService < BaseService + include Gitlab::Utils::StrongMemoize + + def execute + return error('Version is empty.', 400) if version.blank? + return error('Package already exists.', 403) if current_package_exists? + + ActiveRecord::Base.transaction { create_package! } + end + + private + + def create_package! + package = project.packages.create!( + name: name, + version: version, + package_type: 'npm' + ) + + if build.present? + package.create_build_info!(pipeline: build.pipeline) + end + + ::Packages::CreatePackageFileService.new(package, file_params).execute + ::Packages::CreateDependencyService.new(package, package_dependencies).execute + ::Packages::Npm::CreateTagService.new(package, dist_tag).execute + + package + end + + def current_package_exists? + project.packages + .npm + .with_name(name) + .with_version(version) + .exists? + end + + def name + params[:name] + end + + def version + strong_memoize(:version) do + params[:versions].each_key.first + end + end + + def version_data + params[:versions][version] + end + + def build + params[:build] + end + + def dist_tag + params['dist-tags'].each_key.first + end + + def package_file_name + strong_memoize(:package_file_name) do + "#{name}-#{version}.tgz" + end + end + + def attachment + strong_memoize(:attachment) do + params['_attachments'][package_file_name] + end + end + + def file_params + { + file: CarrierWaveStringFile.new(Base64.decode64(attachment['data'])), + size: attachment['length'], + file_sha1: version_data[:dist][:shasum], + file_name: package_file_name + } + end + + def package_dependencies + _version, versions_data = params[:versions].first + versions_data + end + end + end +end diff --git a/app/services/packages/npm/create_tag_service.rb b/app/services/packages/npm/create_tag_service.rb new file mode 100644 index 00000000000..82974d0ca4b --- /dev/null +++ b/app/services/packages/npm/create_tag_service.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true +module Packages + module Npm + class CreateTagService + include Gitlab::Utils::StrongMemoize + + attr_reader :package, :tag_name + + def initialize(package, tag_name) + @package = package + @tag_name = tag_name + end + + def execute + if existing_tag.present? + existing_tag.update_column(:package_id, package.id) + existing_tag + else + package.tags.create!(name: tag_name) + end + end + + private + + def existing_tag + strong_memoize(:existing_tag) do + Packages::TagsFinder + .new(package.project, package.name, package_type: package.package_type) + .find_by_name(tag_name) + end + end + end + end +end diff --git a/app/services/packages/nuget/create_dependency_service.rb b/app/services/packages/nuget/create_dependency_service.rb new file mode 100644 index 00000000000..2be5db732f6 --- /dev/null +++ b/app/services/packages/nuget/create_dependency_service.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true +module Packages + module Nuget + class CreateDependencyService < BaseService + def initialize(package, dependencies = []) + @package = package + @dependencies = dependencies + end + + def execute + return if @dependencies.empty? + + @package.transaction do + create_dependency_links + create_dependency_link_metadata + end + end + + private + + def create_dependency_links + ::Packages::CreateDependencyService + .new(@package, dependencies_for_create_dependency_service) + .execute + end + + def create_dependency_link_metadata + inserted_links = ::Packages::DependencyLink.preload_dependency + .for_package(@package) + + return if inserted_links.empty? + + rows = inserted_links.map do |dependency_link| + raw_dependency = raw_dependency_for(dependency_link.dependency) + + next if raw_dependency[:target_framework].blank? + + { + dependency_link_id: dependency_link.id, + target_framework: raw_dependency[:target_framework] + } + end + + ::Gitlab::Database.bulk_insert(::Packages::Nuget::DependencyLinkMetadatum.table_name, rows.compact) + end + + def raw_dependency_for(dependency) + name = dependency.name + version = dependency.version_pattern.presence + + @dependencies.find do |raw_dependency| + raw_dependency[:name] == name && raw_dependency[:version] == version + end + end + + def dependencies_for_create_dependency_service + names_and_versions = @dependencies.map do |dependency| + [dependency[:name], version_or_empty_string(dependency[:version])] + end.to_h + + { 'dependencies' => names_and_versions } + end + + def version_or_empty_string(version) + return '' if version.blank? + + version + end + end + end +end diff --git a/app/services/packages/nuget/create_package_service.rb b/app/services/packages/nuget/create_package_service.rb new file mode 100644 index 00000000000..68ad7f028e4 --- /dev/null +++ b/app/services/packages/nuget/create_package_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class CreatePackageService < BaseService + TEMPORARY_PACKAGE_NAME = 'NuGet.Temporary.Package' + PACKAGE_VERSION = '0.0.0' + + def execute + project.packages.nuget.create!( + name: TEMPORARY_PACKAGE_NAME, + version: "#{PACKAGE_VERSION}-#{uuid}" + ) + end + + private + + def uuid + SecureRandom.uuid + end + end + end +end diff --git a/app/services/packages/nuget/metadata_extraction_service.rb b/app/services/packages/nuget/metadata_extraction_service.rb new file mode 100644 index 00000000000..6fec398fab0 --- /dev/null +++ b/app/services/packages/nuget/metadata_extraction_service.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class MetadataExtractionService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + XPATHS = { + package_name: '//xmlns:package/xmlns:metadata/xmlns:id', + package_version: '//xmlns:package/xmlns:metadata/xmlns:version', + license_url: '//xmlns:package/xmlns:metadata/xmlns:licenseUrl', + project_url: '//xmlns:package/xmlns:metadata/xmlns:projectUrl', + icon_url: '//xmlns:package/xmlns:metadata/xmlns:iconUrl' + }.freeze + + XPATH_DEPENDENCIES = '//xmlns:package/xmlns:metadata/xmlns:dependencies/xmlns:dependency' + XPATH_DEPENDENCY_GROUPS = '//xmlns:package/xmlns:metadata/xmlns:dependencies/xmlns:group' + XPATH_TAGS = '//xmlns:package/xmlns:metadata/xmlns:tags' + + MAX_FILE_SIZE = 4.megabytes.freeze + + def initialize(package_file_id) + @package_file_id = package_file_id + end + + def execute + raise ExtractionError.new('invalid package file') unless valid_package_file? + + extract_metadata(nuspec_file) + end + + private + + def package_file + strong_memoize(:package_file) do + ::Packages::PackageFile.find_by_id(@package_file_id) + end + end + + def valid_package_file? + package_file && + package_file.package&.nuget? && + package_file.file.size.positive? + end + + def extract_metadata(file) + doc = Nokogiri::XML(file) + + XPATHS.transform_values { |query| doc.xpath(query).text.presence } + .compact + .tap do |metadata| + metadata[:package_dependencies] = extract_dependencies(doc) + metadata[:package_tags] = extract_tags(doc) + end + end + + def extract_dependencies(doc) + dependencies = [] + + doc.xpath(XPATH_DEPENDENCIES).each do |node| + dependencies << extract_dependency(node) + end + + doc.xpath(XPATH_DEPENDENCY_GROUPS).each do |group_node| + target_framework = group_node.attr("targetFramework") + + group_node.xpath("xmlns:dependency").each do |node| + dependencies << extract_dependency(node).merge(target_framework: target_framework) + end + end + + dependencies + end + + def extract_dependency(node) + { + name: node.attr('id'), + version: node.attr('version') + }.compact + end + + def extract_tags(doc) + tags = doc.xpath(XPATH_TAGS).text + + return [] if tags.blank? + + tags.split(::Packages::Tag::NUGET_TAGS_SEPARATOR) + end + + def nuspec_file + package_file.file.use_file do |file_path| + Zip::File.open(file_path) do |zip_file| + entry = zip_file.glob('*.nuspec').first + + raise ExtractionError.new('nuspec file not found') unless entry + raise ExtractionError.new('nuspec file too big') if entry.size > MAX_FILE_SIZE + + entry.get_input_stream.read + end + end + end + end + end +end diff --git a/app/services/packages/nuget/search_service.rb b/app/services/packages/nuget/search_service.rb new file mode 100644 index 00000000000..f7e09e11819 --- /dev/null +++ b/app/services/packages/nuget/search_service.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class SearchService < BaseService + include Gitlab::Utils::StrongMemoize + include ActiveRecord::ConnectionAdapters::Quoting + + MAX_PER_PAGE = 30 + MAX_VERSIONS_PER_PACKAGE = 10 + PRE_RELEASE_VERSION_MATCHING_TERM = '%-%' + + DEFAULT_OPTIONS = { + include_prerelease_versions: true, + per_page: Kaminari.config.default_per_page, + padding: 0 + }.freeze + + def initialize(project, search_term, options = {}) + @project = project + @search_term = search_term + @options = DEFAULT_OPTIONS.merge(options) + + raise ArgumentError, 'negative per_page' if per_page.negative? + raise ArgumentError, 'negative padding' if padding.negative? + end + + def execute + OpenStruct.new( + total_count: package_names.total_count, + results: search_packages + ) + end + + private + + def search_packages + # custom query to get package names and versions as expected from the nuget search api + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24182#technical-notes + # and https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource + subquery_name = :partition_subquery + arel_table = Arel::Table.new(:partition_subquery) + column_names = Packages::Package.column_names.map do |cn| + "#{subquery_name}.#{quote_column_name(cn)}" + end + + # rubocop: disable CodeReuse/ActiveRecord + pkgs = Packages::Package.select(column_names.join(',')) + .from(package_names_partition, subquery_name) + .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE)) + + return pkgs if include_prerelease_versions? + + # we can't use pkgs.without_version_like since we have a custom from + pkgs.where.not(arel_table[:version].matches(PRE_RELEASE_VERSION_MATCHING_TERM)) + end + + def package_names_partition + table_name = quote_table_name(Packages::Package.table_name) + name_column = "#{table_name}.#{quote_column_name('name')}" + created_at_column = "#{table_name}.#{quote_column_name('created_at')}" + select_sql = "ROW_NUMBER() OVER (PARTITION BY #{name_column} ORDER BY #{created_at_column} DESC) AS row_number, #{table_name}.*" + + @project.packages + .select(select_sql) + .nuget + .has_version + .without_nuget_temporary_name + .with_name(package_names) + end + + def package_names + strong_memoize(:package_names) do + pkgs = @project.packages + .nuget + .has_version + .without_nuget_temporary_name + .order_name + .select_distinct_name + pkgs = pkgs.without_version_like(PRE_RELEASE_VERSION_MATCHING_TERM) unless include_prerelease_versions? + pkgs = pkgs.search_by_name(@search_term) if @search_term.present? + pkgs.page(0) # we're using a padding + .per(per_page) + .padding(padding) + end + end + + def include_prerelease_versions? + @options[:include_prerelease_versions] + end + + def padding + @options[:padding] + end + + def per_page + [@options[:per_page], MAX_PER_PAGE].min + end + end + end +end diff --git a/app/services/packages/nuget/sync_metadatum_service.rb b/app/services/packages/nuget/sync_metadatum_service.rb new file mode 100644 index 00000000000..ca9cc4d5b78 --- /dev/null +++ b/app/services/packages/nuget/sync_metadatum_service.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class SyncMetadatumService + include Gitlab::Utils::StrongMemoize + + def initialize(package, metadata) + @package = package + @metadata = metadata + end + + def execute + if blank_metadata? + metadatum.destroy! if metadatum.persisted? + else + metadatum.update!( + license_url: license_url, + project_url: project_url, + icon_url: icon_url + ) + end + end + + private + + def metadatum + strong_memoize(:metadatum) do + @package.nuget_metadatum || @package.build_nuget_metadatum + end + end + + def blank_metadata? + project_url.blank? && license_url.blank? && icon_url.blank? + end + + def project_url + @metadata[:project_url] + end + + def license_url + @metadata[:license_url] + end + + def icon_url + @metadata[:icon_url] + end + end + end +end diff --git a/app/services/packages/nuget/update_package_from_metadata_service.rb b/app/services/packages/nuget/update_package_from_metadata_service.rb new file mode 100644 index 00000000000..f72b1386985 --- /dev/null +++ b/app/services/packages/nuget/update_package_from_metadata_service.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class UpdatePackageFromMetadataService + include Gitlab::Utils::StrongMemoize + include ExclusiveLeaseGuard + + # used by ExclusiveLeaseGuard + DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze + + InvalidMetadataError = Class.new(StandardError) + + def initialize(package_file) + @package_file = package_file + end + + def execute + raise InvalidMetadataError.new('package name and/or package version not found in metadata') unless valid_metadata? + + try_obtain_lease do + @package_file.transaction do + package = existing_package ? link_to_existing_package : update_linked_package + + update_package(package) + + # Updating file_name updates the path where the file is stored. + # We must pass the file again so that CarrierWave can handle the update + @package_file.update!( + file_name: package_filename, + file: @package_file.file + ) + end + end + end + + private + + def update_package(package) + ::Packages::Nuget::SyncMetadatumService + .new(package, metadata.slice(:project_url, :license_url, :icon_url)) + .execute + ::Packages::UpdateTagsService + .new(package, package_tags) + .execute + rescue => e + raise InvalidMetadataError, e.message + end + + def valid_metadata? + package_name.present? && package_version.present? + end + + def link_to_existing_package + package_to_destroy = @package_file.package + # Updating package_id updates the path where the file is stored. + # We must pass the file again so that CarrierWave can handle the update + @package_file.update!( + package_id: existing_package.id, + file: @package_file.file + ) + package_to_destroy.destroy! + existing_package + end + + def update_linked_package + @package_file.package.update!( + name: package_name, + version: package_version + ) + + ::Packages::Nuget::CreateDependencyService.new(@package_file.package, package_dependencies) + .execute + @package_file.package + end + + def existing_package + strong_memoize(:existing_package) do + @package_file.project.packages + .nuget + .with_name(package_name) + .with_version(package_version) + .first + end + end + + def package_name + metadata[:package_name] + end + + def package_version + metadata[:package_version] + end + + def package_dependencies + metadata.fetch(:package_dependencies, []) + end + + def package_tags + metadata.fetch(:package_tags, []) + end + + def metadata + strong_memoize(:metadata) do + ::Packages::Nuget::MetadataExtractionService.new(@package_file.id).execute + end + end + + def package_filename + "#{package_name.downcase}.#{package_version.downcase}.nupkg" + end + + # used by ExclusiveLeaseGuard + def lease_key + package_id = existing_package ? existing_package.id : @package_file.package_id + "packages:nuget:update_package_from_metadata_service:package:#{package_id}" + end + + # used by ExclusiveLeaseGuard + def lease_timeout + DEFAULT_LEASE_TIMEOUT + end + end + end +end diff --git a/app/services/packages/pypi/create_package_service.rb b/app/services/packages/pypi/create_package_service.rb new file mode 100644 index 00000000000..1313fc80e33 --- /dev/null +++ b/app/services/packages/pypi/create_package_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Packages + module Pypi + class CreatePackageService < BaseService + include ::Gitlab::Utils::StrongMemoize + + def execute + ::Packages::Package.transaction do + Packages::Pypi::Metadatum.upsert( + package_id: created_package.id, + required_python: params[:requires_python] + ) + + ::Packages::CreatePackageFileService.new(created_package, file_params).execute + end + end + + private + + def created_package + strong_memoize(:created_package) do + project + .packages + .pypi + .safe_find_or_create_by!(name: params[:name], version: params[:version]) + end + end + + def file_params + { + file: params[:content], + file_name: params[:content].original_filename, + file_md5: params[:md5_digest], + file_sha256: params[:sha256_digest] + } + end + end + end +end diff --git a/app/services/packages/remove_tag_service.rb b/app/services/packages/remove_tag_service.rb new file mode 100644 index 00000000000..465b85506a6 --- /dev/null +++ b/app/services/packages/remove_tag_service.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +module Packages + class RemoveTagService < BaseService + attr_reader :package_tag + + def initialize(package_tag) + raise ArgumentError, "Package tag must be set" if package_tag.blank? + + @package_tag = package_tag + end + + def execute + package_tag.delete + end + end +end diff --git a/app/services/packages/update_tags_service.rb b/app/services/packages/update_tags_service.rb new file mode 100644 index 00000000000..da50cd3479e --- /dev/null +++ b/app/services/packages/update_tags_service.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +module Packages + class UpdateTagsService + include Gitlab::Utils::StrongMemoize + + def initialize(package, tags = []) + @package = package + @tags = tags + end + + def execute + return if @tags.empty? + + tags_to_destroy = existing_tags - @tags + tags_to_create = @tags - existing_tags + + @package.tags.with_name(tags_to_destroy).delete_all if tags_to_destroy.any? + ::Gitlab::Database.bulk_insert(Packages::Tag.table_name, rows(tags_to_create)) if tags_to_create.any? + end + + private + + def existing_tags + strong_memoize(:existing_tags) do + @package.tag_names + end + end + + def rows(tags) + now = Time.zone.now + tags.map do |tag| + { + package_id: @package.id, + name: tag, + created_at: now, + updated_at: now + } + end + end + end +end diff --git a/app/services/personal_access_tokens/last_used_service.rb b/app/services/personal_access_tokens/last_used_service.rb new file mode 100644 index 00000000000..9066fd1acdf --- /dev/null +++ b/app/services/personal_access_tokens/last_used_service.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module PersonalAccessTokens + class LastUsedService + def initialize(personal_access_token) + @personal_access_token = personal_access_token + end + + def execute + # Needed to avoid calling service on Oauth tokens + return unless @personal_access_token.has_attribute?(:last_used_at) + + # We _only_ want to update last_used_at and not also updated_at (which + # would be updated when using #touch). + @personal_access_token.update_column(:last_used_at, Time.zone.now) if update? + end + + private + + def update? + return false if ::Gitlab::Database.read_only? + + last_used = @personal_access_token.last_used_at + + last_used.nil? || (last_used <= 1.day.ago) + end + end +end diff --git a/app/services/post_receive_service.rb b/app/services/post_receive_service.rb index 65e6ebc17d2..69c9868c75c 100644 --- a/app/services/post_receive_service.rb +++ b/app/services/post_receive_service.rb @@ -29,8 +29,6 @@ class PostReceiveService response.add_alert_message(message) end - response.add_alert_message(storage_size_limit_alert) - broadcast_message = BroadcastMessage.current_banner_messages&.last&.message response.add_alert_message(broadcast_message) @@ -76,19 +74,6 @@ class PostReceiveService ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end - - private - - def storage_size_limit_alert - return unless repository&.repo_type&.project? - - payload = Namespaces::CheckStorageSizeService.new(project.namespace, user).execute.payload - return unless payload.present? - - alert_level = "##### #{payload[:alert_level].to_s.upcase} #####" - - [alert_level, payload[:usage_message], payload[:explanation_message]].join("\n") - end end PostReceiveService.prepend_if_ee('EE::PostReceiveService') diff --git a/app/services/projects/after_import_service.rb b/app/services/projects/after_import_service.rb index fad2290a47b..b37ae56ba0f 100644 --- a/app/services/projects/after_import_service.rb +++ b/app/services/projects/after_import_service.rb @@ -26,7 +26,7 @@ module Projects message: 'Project housekeeping failed', project_full_path: @project.full_path, project_id: @project.id, - error: e.message + 'error.message' => e.message ) end diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb index 86c408aeec8..e08bc8efb15 100644 --- a/app/services/projects/alerting/notify_service.rb +++ b/app/services/projects/alerting/notify_service.rb @@ -4,7 +4,7 @@ module Projects module Alerting class NotifyService < BaseService include Gitlab::Utils::StrongMemoize - include IncidentManagement::Settings + include ::IncidentManagement::Settings def execute(token) return forbidden unless alerts_service_activated? @@ -55,7 +55,7 @@ module Projects def find_alert_by_fingerprint(fingerprint) return unless fingerprint - AlertManagement::Alert.for_fingerprint(project, fingerprint).first + AlertManagement::Alert.not_resolved.for_fingerprint(project, fingerprint).first end def send_email? @@ -65,8 +65,7 @@ module Projects def process_incident_issues(alert) return if alert.issue - IncidentManagement::ProcessAlertWorker - .perform_async(project.id, parsed_payload, alert.id) + ::IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id) end def send_alert_email @@ -76,7 +75,7 @@ module Projects end def parsed_payload - Gitlab::Alerting::NotificationPayloadParser.call(params.to_h) + Gitlab::Alerting::NotificationPayloadParser.call(params.to_h, project) end def valid_token?(token) diff --git a/app/services/projects/batch_forks_count_service.rb b/app/services/projects/batch_forks_count_service.rb index 6467744a435..d12772b40ff 100644 --- a/app/services/projects/batch_forks_count_service.rb +++ b/app/services/projects/batch_forks_count_service.rb @@ -5,6 +5,21 @@ # because the service use maps to retrieve the project ids module Projects class BatchForksCountService < Projects::BatchCountService + def refresh_cache_and_retrieve_data + count_services = @projects.map { |project| count_service.new(project) } + + values = Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Rails.cache.fetch_multi(*(count_services.map { |ser| ser.cache_key } )) { |key| nil } + end + + results_per_service = Hash[count_services.zip(values.values)] + projects_to_refresh = results_per_service.select { |_k, value| value.nil? } + projects_to_refresh = recreate_cache(projects_to_refresh) + + results_per_service.update(projects_to_refresh) + results_per_service.transform_keys { |k| k.project } + end + # rubocop: disable CodeReuse/ActiveRecord def global_count @global_count ||= begin @@ -18,5 +33,13 @@ module Projects def count_service ::Projects::ForksCountService end + + def recreate_cache(projects_to_refresh) + projects_to_refresh.each_with_object({}) do |(service, _v), hash| + count = global_count[service.project.id].to_i + service.refresh_cache { count } + hash[service] = count + end + end end end diff --git a/app/services/projects/container_repository/delete_tags_service.rb b/app/services/projects/container_repository/delete_tags_service.rb index 21081bd077f..5d4059710bb 100644 --- a/app/services/projects/container_repository/delete_tags_service.rb +++ b/app/services/projects/container_repository/delete_tags_service.rb @@ -3,6 +3,8 @@ module Projects module ContainerRepository class DeleteTagsService < BaseService + LOG_DATA_BASE = { service_class: self.to_s }.freeze + def execute(container_repository) return error('access denied') unless can?(current_user, :destroy_container_image, project) @@ -51,10 +53,27 @@ module Projects def smart_delete(container_repository, tag_names) fast_delete_enabled = Feature.enabled?(:container_registry_fast_tag_delete, default_enabled: true) - if fast_delete_enabled && container_repository.client.supports_tag_delete? - fast_delete(container_repository, tag_names) + response = if fast_delete_enabled && container_repository.client.supports_tag_delete? + fast_delete(container_repository, tag_names) + else + slow_delete(container_repository, tag_names) + end + + response.tap { |r| log_response(r, container_repository) } + end + + def log_response(response, container_repository) + log_data = LOG_DATA_BASE.merge( + container_repository_id: container_repository.id, + message: 'deleted tags' + ) + + if response[:status] == :success + log_data[:deleted_tags_count] = response[:deleted].size + log_info(log_data) else - slow_delete(container_repository, tag_names) + log_data[:message] = response[:message] + log_error(log_data) end end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index bffd443c49f..6569277ad9d 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -84,8 +84,12 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.full_name}\"") + # Skip writing the config for project imports/forks because it + # will always fail since the Git directory doesn't exist until + # a background job creates it (see Project#add_import_job). + @project.write_repository_config unless @project.import? + unless @project.gitlab_project_import? - @project.write_repository_config @project.create_wiki unless skip_wiki? end @@ -103,12 +107,13 @@ module Projects create_readme if @initialize_with_readme end - # Refresh the current user's authorizations inline (so they can access the - # project immediately after this request completes), and any other affected - # users in the background + # Add an authorization for the current user authorizations inline + # (so they can access the project immediately after this request + # completes), and any other affected users in the background def setup_authorizations if @project.group - current_user.refresh_authorized_projects + current_user.project_authorizations.create!(project: @project, + access_level: @project.group.max_member_access_for_user(current_user)) if Feature.enabled?(:specialized_project_authorization_workers) AuthorizedProjectUpdate::ProjectCreateWorker.perform_async(@project.id) @@ -131,7 +136,7 @@ module Projects def create_readme commit_attrs = { - branch_name: 'master', + branch_name: Gitlab::CurrentSettings.default_branch_name.presence || 'master', commit_message: 'Initial commit', file_path: 'README.md', file_content: "# #{@project.name}\n\n#{@project.description}" diff --git a/app/services/projects/forks_count_service.rb b/app/services/projects/forks_count_service.rb index ca85e2dc281..848d8d54104 100644 --- a/app/services/projects/forks_count_service.rb +++ b/app/services/projects/forks_count_service.rb @@ -3,6 +3,8 @@ module Projects # Service class for getting and caching the number of forks of a project. class ForksCountService < Projects::CountService + attr_reader :project + def cache_key_name 'forks_count' end diff --git a/app/services/projects/group_links/create_service.rb b/app/services/projects/group_links/create_service.rb index 2ba3cd6694f..3c3cab26fb5 100644 --- a/app/services/projects/group_links/create_service.rb +++ b/app/services/projects/group_links/create_service.rb @@ -13,12 +13,32 @@ module Projects ) if link.save - group.refresh_members_authorized_projects + setup_authorizations(group) success(link: link) else error(link.errors.full_messages.to_sentence, 409) end end + + private + + def setup_authorizations(group) + if Feature.enabled?(:specialized_project_authorization_project_share_worker) + AuthorizedProjectUpdate::ProjectGroupLinkCreateWorker.perform_async(project.id, group.id) + + # AuthorizedProjectsWorker uses an exclusive lease per user but + # specialized workers might have synchronization issues. Until we + # compare the inconsistency rates of both approaches, we still run + # AuthorizedProjectsWorker but with some delay and lower urgency as a + # safety net. + group.refresh_members_authorized_projects( + blocking: false, + priority: UserProjectAccessChangedService::LOW_PRIORITY + ) + else + group.refresh_members_authorized_projects(blocking: false) + end + end end end end diff --git a/app/services/projects/operations/update_service.rb b/app/services/projects/operations/update_service.rb index 7aa7ea73639..7af489c3751 100644 --- a/app/services/projects/operations/update_service.rb +++ b/app/services/projects/operations/update_service.rb @@ -108,7 +108,18 @@ module Projects end def incident_management_setting_params - params.slice(:incident_management_setting_attributes) + attrs = params[:incident_management_setting_attributes] + return {} unless attrs + + regenerate_token = attrs.delete(:regenerate_token) + + if regenerate_token + attrs[:pagerduty_token] = nil + else + attrs = attrs.except(:pagerduty_token) + end + + { incident_management_setting_attributes: attrs } end end end diff --git a/app/services/projects/prometheus/alerts/create_events_service.rb b/app/services/projects/prometheus/alerts/create_events_service.rb deleted file mode 100644 index 4fcf841314b..00000000000 --- a/app/services/projects/prometheus/alerts/create_events_service.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -module Projects - module Prometheus - module Alerts - # Persists a series of Prometheus alert events as list of PrometheusAlertEvent. - class CreateEventsService < BaseService - def execute - create_events_from(alerts) - end - - private - - def create_events_from(alerts) - Array.wrap(alerts).map { |alert| create_event(alert) }.compact - end - - def create_event(payload) - parsed_alert = Gitlab::Alerting::Alert.new(project: project, payload: payload) - - return unless parsed_alert.valid? - - if parsed_alert.gitlab_managed? - create_managed_prometheus_alert_event(parsed_alert) - else - create_self_managed_prometheus_alert_event(parsed_alert) - end - end - - def alerts - params['alerts'] - end - - def find_alert(metric) - Projects::Prometheus::AlertsFinder - .new(project: project, metric: metric) - .execute - .first - end - - def create_managed_prometheus_alert_event(parsed_alert) - alert = find_alert(parsed_alert.metric_id) - event = PrometheusAlertEvent.find_or_initialize_by_payload_key(parsed_alert.project, alert, parsed_alert.gitlab_fingerprint) - - set_status(parsed_alert, event) - end - - def create_self_managed_prometheus_alert_event(parsed_alert) - event = SelfManagedPrometheusAlertEvent.find_or_initialize_by_payload_key(parsed_alert.project, parsed_alert.gitlab_fingerprint) do |event| - event.environment = parsed_alert.environment - event.title = parsed_alert.title - event.query_expression = parsed_alert.full_query - end - - set_status(parsed_alert, event) - end - - def set_status(parsed_alert, event) - persisted = case parsed_alert.status - when 'firing' - event.fire(parsed_alert.starts_at) - when 'resolved' - event.resolve(parsed_alert.ends_at) - end - - event if persisted - end - end - end - end -end diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index 877a4f99a94..ea557ebe20f 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -5,7 +5,7 @@ module Projects module Alerts class NotifyService < BaseService include Gitlab::Utils::StrongMemoize - include IncidentManagement::Settings + include ::IncidentManagement::Settings # This set of keys identifies a payload as a valid Prometheus # payload and thus processable by this service. See also @@ -23,9 +23,7 @@ module Projects return unauthorized unless valid_alert_manager_token?(token) process_prometheus_alerts - persist_events send_alert_email if send_email? - process_incident_issues if process_issues? ServiceResponse.success end @@ -132,13 +130,6 @@ module Projects .prometheus_alerts_fired(project, firings) end - def process_incident_issues - alerts.each do |alert| - IncidentManagement::ProcessPrometheusAlertWorker - .perform_async(project.id, alert.to_h) - end - end - def process_prometheus_alerts alerts.each do |alert| AlertManagement::ProcessPrometheusAlertService @@ -147,10 +138,6 @@ module Projects end end - def persist_events - CreateEventsService.new(project, nil, params).execute - end - def bad_request ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) end diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb index 4adcda042d1..b6465810fde 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_service_template.rb @@ -26,7 +26,7 @@ module Projects def propagate_projects_with_template loop do - batch = Project.uncached { project_ids_without_integration } + batch = Project.uncached { Project.ids_without_integration(template, BATCH_SIZE) } bulk_create_from_template(batch) unless batch.empty? @@ -50,22 +50,6 @@ module Projects end end - # rubocop: disable CodeReuse/ActiveRecord - def project_ids_without_integration - services = Service - .select('1') - .where('services.project_id = projects.id') - .where(type: template.type) - - Project - .where('NOT EXISTS (?)', services) - .where(pending_delete: false) - .where(archived: false) - .limit(BATCH_SIZE) - .pluck(:id) - end - # rubocop: enable CodeReuse/ActiveRecord - def bulk_insert(klass, columns, values_array) items_to_insert = values_array.map { |array| Hash[columns.zip(array)] } diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 5f8ef75a8d7..d6c0d647468 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -29,7 +29,7 @@ module Projects remote_mirror.ensure_remote! # https://gitlab.com/gitlab-org/gitaly/-/issues/2670 - if Feature.disabled?(:gitaly_ruby_remote_branches_ls_remote) + if Feature.disabled?(:gitaly_ruby_remote_branches_ls_remote, default_enabled: true) repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true) end diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb index fa8d4c5aa5f..7b346c09635 100644 --- a/app/services/projects/update_repository_storage_service.rb +++ b/app/services/projects/update_repository_storage_service.rb @@ -14,7 +14,11 @@ module Projects end def execute - repository_storage_move.start! + repository_storage_move.with_lock do + return ServiceResponse.success unless repository_storage_move.scheduled? # rubocop:disable Cop/AvoidReturnFromBlocks + + repository_storage_move.start! + end raise SameFilesystemError if same_filesystem?(repository.storage, destination_storage_name) @@ -79,8 +83,6 @@ module Projects full_path ) - new_repository.create_repository - new_repository.replicate(raw_repository) new_checksum = new_repository.checksum @@ -93,25 +95,25 @@ module Projects old_repository_storage = project.repository_storage new_project_path = moved_path(project.disk_path) - # Notice that the block passed to `run_after_commit` will run with `project` + # Notice that the block passed to `run_after_commit` will run with `repository_storage_move` # as its context - project.run_after_commit do + repository_storage_move.run_after_commit do GitlabShellWorker.perform_async(:mv_repository, old_repository_storage, - disk_path, + project.disk_path, new_project_path) - if wiki.repository_exists? + if project.wiki.repository_exists? GitlabShellWorker.perform_async(:mv_repository, old_repository_storage, - wiki.disk_path, + project.wiki.disk_path, "#{new_project_path}.wiki") end - if design_repository.exists? + if project.design_repository.exists? GitlabShellWorker.perform_async(:mv_repository, old_repository_storage, - design_repository.disk_path, + project.design_repository.disk_path, "#{new_project_path}.design") end end diff --git a/app/services/prometheus/proxy_service.rb b/app/services/prometheus/proxy_service.rb index e0bc5518d30..33635796771 100644 --- a/app/services/prometheus/proxy_service.rb +++ b/app/services/prometheus/proxy_service.rb @@ -22,16 +22,20 @@ module Prometheus attr_accessor :proxyable, :method, :path, :params + PROMETHEUS_QUERY_API = 'query' + PROMETHEUS_QUERY_RANGE_API = 'query_range' + PROMETHEUS_SERIES_API = 'series' + PROXY_SUPPORT = { - 'query' => { + PROMETHEUS_QUERY_API => { method: ['GET'], params: %w(query time timeout) }, - 'query_range' => { + PROMETHEUS_QUERY_RANGE_API => { method: ['GET'], params: %w(query start end step timeout) }, - 'series' => { + PROMETHEUS_SERIES_API => { method: %w(GET), params: %w(match start end) } diff --git a/app/services/prometheus/proxy_variable_substitution_service.rb b/app/services/prometheus/proxy_variable_substitution_service.rb index 10fb3a8c1b5..820b551c30a 100644 --- a/app/services/prometheus/proxy_variable_substitution_service.rb +++ b/app/services/prometheus/proxy_variable_substitution_service.rb @@ -19,10 +19,52 @@ module Prometheus :substitute_params, :substitute_variables + # @param environment [Environment] + # @param params [Hash<Symbol,Any>] + # @param params - query [String] The Prometheus query string. + # @param params - start [String] (optional) A time string in the rfc3339 format. + # @param params - start_time [String] (optional) A time string in the rfc3339 format. + # @param params - end [String] (optional) A time string in the rfc3339 format. + # @param params - end_time [String] (optional) A time string in the rfc3339 format. + # @param params - variables [ActionController::Parameters] (optional) Variables with their values. + # The keys in the Hash should be the name of the variable. The value should be the value of the + # variable. Ex: `ActionController::Parameters.new(variable1: 'value 1', variable2: 'value 2').permit!` + # @return [Prometheus::ProxyVariableSubstitutionService] + # + # Example: + # Prometheus::ProxyVariableSubstitutionService.new(environment, { + # params: { + # start_time: '2020-07-03T06:08:36Z', + # end_time: '2020-07-03T14:08:52Z', + # query: 'up{instance="{{instance}}"}', + # variables: { instance: 'srv1' } + # } + # }) def initialize(environment, params = {}) @environment, @params = environment, params.deep_dup end + # @return - params [Hash<Symbol,Any>] Returns a Hash containing a params key which is + # similar to the `params` that is passed to the initialize method with 2 differences: + # 1. Variables in the query string are substituted with their values. + # If a variable present in the query string has no known value (values + # are obtained from the `variables` Hash in `params` or from + # `Gitlab::Prometheus::QueryVariables.call`), it will not be substituted. + # 2. `start` and `end` keys are added, with their values copied from `start_time` + # and `end_time`. + # + # Example output: + # + # { + # params: { + # start_time: '2020-07-03T06:08:36Z', + # start: '2020-07-03T06:08:36Z', + # end_time: '2020-07-03T14:08:52Z', + # end: '2020-07-03T14:08:52Z', + # query: 'up{instance="srv1"}', + # variables: { instance: 'srv1' } + # } + # } def execute execute_steps end diff --git a/app/services/releases/create_evidence_service.rb b/app/services/releases/create_evidence_service.rb index ac13dce1729..9c370722d2c 100644 --- a/app/services/releases/create_evidence_service.rb +++ b/app/services/releases/create_evidence_service.rb @@ -10,7 +10,7 @@ module Releases def execute evidence = release.evidences.build - summary = Evidences::EvidenceSerializer.new.represent(evidence) # rubocop: disable CodeReuse/Serializer + summary = ::Evidences::EvidenceSerializer.new.represent(evidence, evidence_options) # rubocop: disable CodeReuse/Serializer evidence.summary = summary # TODO: fix the sha generating https://gitlab.com/gitlab-org/gitlab/-/issues/209000 evidence.summary_sha = Gitlab::CryptoHelper.sha256(summary) @@ -20,6 +20,12 @@ module Releases private - attr_reader :release + attr_reader :release, :pipeline + + def evidence_options + {} + end end end + +Releases::CreateEvidenceService.prepend_if_ee('EE::Releases::CreateEvidenceService') diff --git a/app/services/repositories/base_service.rb b/app/services/repositories/base_service.rb index a99a65b7edb..efb6f6de8db 100644 --- a/app/services/repositories/base_service.rb +++ b/app/services/repositories/base_service.rb @@ -8,20 +8,19 @@ class Repositories::BaseService < BaseService attr_reader :repository delegate :container, :disk_path, :full_path, to: :repository - delegate :repository_storage, to: :container def initialize(repository) @repository = repository end def repo_exists?(path) - gitlab_shell.repository_exists?(repository_storage, path + '.git') + gitlab_shell.repository_exists?(repository.shard, path + '.git') end def mv_repository(from_path, to_path) return true unless repo_exists?(from_path) - gitlab_shell.mv_repository(repository_storage, from_path, to_path) + gitlab_shell.mv_repository(repository.shard, from_path, to_path) end # Build a path for removing repositories diff --git a/app/services/repositories/destroy_service.rb b/app/services/repositories/destroy_service.rb index b12d0744387..1e34dfbe398 100644 --- a/app/services/repositories/destroy_service.rb +++ b/app/services/repositories/destroy_service.rb @@ -14,8 +14,17 @@ class Repositories::DestroyService < Repositories::BaseService log_info(%Q{Repository "#{disk_path}" moved to "#{removal_path}" for repository "#{full_path}"}) current_repository = repository - container.run_after_commit do + + # Because GitlabShellWorker is inside a run_after_commit callback it will + # never be triggered on a read-only instance. + # + # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/223272 + if Gitlab::Database.read_only? Repositories::ShellDestroyService.new(current_repository).execute + else + container.run_after_commit do + Repositories::ShellDestroyService.new(current_repository).execute + end end log_info("Repository \"#{full_path}\" was removed") diff --git a/app/services/repositories/shell_destroy_service.rb b/app/services/repositories/shell_destroy_service.rb index 2f5af10e24c..d25cb28c6d7 100644 --- a/app/services/repositories/shell_destroy_service.rb +++ b/app/services/repositories/shell_destroy_service.rb @@ -9,7 +9,7 @@ class Repositories::ShellDestroyService < Repositories::BaseService GitlabShellWorker.perform_in(delay, :remove_repository, - repository_storage, + repository.shard, removal_path) end end diff --git a/app/services/resource_access_tokens/create_service.rb b/app/services/resource_access_tokens/create_service.rb index c8e86e68383..2d0a78feb8e 100644 --- a/app/services/resource_access_tokens/create_service.rb +++ b/app/services/resource_access_tokens/create_service.rb @@ -13,8 +13,6 @@ module ResourceAccessTokens return unless feature_enabled? return error("User does not have permission to create #{resource_type} Access Token") unless has_permission_to_create? - # We skip authorization by default, since the user creating the bot is not an admin - # and project/group bot users are not created via sign-up user = create_user return error(user.errors.full_messages.to_sentence) unless user.persisted? @@ -49,6 +47,11 @@ module ResourceAccessTokens end def create_user + # Even project maintainers can create project access tokens, which in turn + # creates a bot user, and so it becomes necessary to have `skip_authorization: true` + # since someone like a project maintainer does not inherently have the ability + # to create a new user in the system. + Users::CreateService.new(current_user, default_user_params).execute(skip_authorization: true) end @@ -57,7 +60,8 @@ module ResourceAccessTokens name: params[:name] || "#{resource.name.to_s.humanize} bot", email: generate_email, username: generate_username, - user_type: "#{resource_type}_bot".to_sym + user_type: "#{resource_type}_bot".to_sym, + skip_confirmation: true # Bot users should always have their emails confirmed. } end diff --git a/app/services/resource_access_tokens/revoke_service.rb b/app/services/resource_access_tokens/revoke_service.rb index eea6bff572b..efeb0bfb8d5 100644 --- a/app/services/resource_access_tokens/revoke_service.rb +++ b/app/services/resource_access_tokens/revoke_service.rb @@ -35,7 +35,7 @@ module ResourceAccessTokens attr_reader :current_user, :access_token, :bot_user, :resource def remove_member - ::Members::DestroyService.new(current_user).execute(find_member) + ::Members::DestroyService.new(current_user).execute(find_member, destroy_bot: true) end def migrate_to_ghost_user diff --git a/app/services/resource_events/base_synthetic_notes_builder_service.rb b/app/services/resource_events/base_synthetic_notes_builder_service.rb index db8bf6e4b74..a2d78ec67c3 100644 --- a/app/services/resource_events/base_synthetic_notes_builder_service.rb +++ b/app/services/resource_events/base_synthetic_notes_builder_service.rb @@ -23,11 +23,25 @@ module ResourceEvents private - def since_fetch_at(events) + def apply_common_filters(events) + events = apply_last_fetched_at(events) + events = apply_fetch_until(events) + + events + end + + def apply_last_fetched_at(events) return events unless params[:last_fetched_at].present? - last_fetched_at = Time.zone.at(params.fetch(:last_fetched_at).to_i) - events.created_after(last_fetched_at - NotesFinder::FETCH_OVERLAP) + last_fetched_at = params[:last_fetched_at] - NotesFinder::FETCH_OVERLAP + + events.created_after(last_fetched_at) + end + + def apply_fetch_until(events) + return events unless params[:fetch_until].present? + + events.created_on_or_before(params[:fetch_until]) end def resource_parent diff --git a/app/services/resource_events/change_state_service.rb b/app/services/resource_events/change_state_service.rb index 8beb76d8aee..202972c1efd 100644 --- a/app/services/resource_events/change_state_service.rb +++ b/app/services/resource_events/change_state_service.rb @@ -8,12 +8,18 @@ module ResourceEvents @user, @resource = user, resource end - def execute(state) + def execute(params) + @params = params + ResourceStateEvent.create( user: user, issue: issue, merge_request: merge_request, + source_commit: commit_id_of(mentionable_source), + source_merge_request_id: merge_request_id_of(mentionable_source), state: ResourceStateEvent.states[state], + close_after_error_tracking_resolve: close_after_error_tracking_resolve, + close_auto_resolve_prometheus_alert: close_auto_resolve_prometheus_alert, created_at: Time.zone.now) resource.expire_note_etag_cache @@ -21,6 +27,36 @@ module ResourceEvents private + attr_reader :params + + def close_auto_resolve_prometheus_alert + params[:close_auto_resolve_prometheus_alert] || false + end + + def close_after_error_tracking_resolve + params[:close_after_error_tracking_resolve] || false + end + + def state + params[:status] + end + + def mentionable_source + params[:mentionable_source] + end + + def commit_id_of(mentionable_source) + return unless mentionable_source.is_a?(Commit) + + mentionable_source.id[0...40] + end + + def merge_request_id_of(mentionable_source) + return unless mentionable_source.is_a?(MergeRequest) + + mentionable_source.id + end + def issue return unless resource.is_a?(Issue) diff --git a/app/services/resource_events/synthetic_label_notes_builder_service.rb b/app/services/resource_events/synthetic_label_notes_builder_service.rb index fd128101b49..5915ea938cf 100644 --- a/app/services/resource_events/synthetic_label_notes_builder_service.rb +++ b/app/services/resource_events/synthetic_label_notes_builder_service.rb @@ -19,7 +19,7 @@ module ResourceEvents return [] unless resource.respond_to?(:resource_label_events) events = resource.resource_label_events.includes(:label, user: :status) # rubocop: disable CodeReuse/ActiveRecord - events = since_fetch_at(events) + events = apply_common_filters(events) events.group_by { |event| event.discussion_id } end diff --git a/app/services/resource_events/synthetic_milestone_notes_builder_service.rb b/app/services/resource_events/synthetic_milestone_notes_builder_service.rb index cc6383d7083..10acf94e22b 100644 --- a/app/services/resource_events/synthetic_milestone_notes_builder_service.rb +++ b/app/services/resource_events/synthetic_milestone_notes_builder_service.rb @@ -19,7 +19,7 @@ module ResourceEvents return [] unless resource.respond_to?(:resource_milestone_events) events = resource.resource_milestone_events.includes(user: :status) # rubocop: disable CodeReuse/ActiveRecord - since_fetch_at(events) + apply_common_filters(events) end end end diff --git a/app/services/resource_events/synthetic_state_notes_builder_service.rb b/app/services/resource_events/synthetic_state_notes_builder_service.rb index 763134d98d8..71d40200365 100644 --- a/app/services/resource_events/synthetic_state_notes_builder_service.rb +++ b/app/services/resource_events/synthetic_state_notes_builder_service.rb @@ -14,7 +14,7 @@ module ResourceEvents return [] unless resource.respond_to?(:resource_state_events) events = resource.resource_state_events.includes(user: :status) # rubocop: disable CodeReuse/ActiveRecord - since_fetch_at(events) + apply_common_filters(events) end end end diff --git a/app/services/service_desk_settings/update_service.rb b/app/services/service_desk_settings/update_service.rb new file mode 100644 index 00000000000..08106b04d18 --- /dev/null +++ b/app/services/service_desk_settings/update_service.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ServiceDeskSettings + class UpdateService < BaseService + def execute + settings = ServiceDeskSetting.safe_find_or_create_by!(project_id: project.id) + + unless ::Feature.enabled?(:service_desk_custom_address, project) + params.delete(:project_key) + end + + if settings.update(params) + success + else + error(settings.errors.full_messages.to_sentence) + end + end + end +end diff --git a/app/services/snippets/base_service.rb b/app/services/snippets/base_service.rb index 5d1fe815d83..d9e8326f159 100644 --- a/app/services/snippets/base_service.rb +++ b/app/services/snippets/base_service.rb @@ -6,13 +6,15 @@ module Snippets CreateRepositoryError = Class.new(StandardError) - attr_reader :uploaded_assets, :snippet_files + attr_reader :uploaded_assets, :snippet_actions def initialize(project, user = nil, params = {}) super @uploaded_assets = Array(@params.delete(:files).presence) - @snippet_files = SnippetInputActionCollection.new(Array(@params.delete(:snippet_files).presence)) + + input_actions = Array(@params.delete(:snippet_actions).presence) + @snippet_actions = SnippetInputActionCollection.new(input_actions, allowed_actions: restricted_files_actions) filter_spam_check_params end @@ -30,18 +32,18 @@ module Snippets end def valid_params? - return true if snippet_files.empty? + return true if snippet_actions.empty? - (params.keys & [:content, :file_name]).none? && snippet_files.valid? + (params.keys & [:content, :file_name]).none? && snippet_actions.valid? end def invalid_params_error(snippet) - if snippet_files.valid? + if snippet_actions.valid? [:content, :file_name].each do |key| snippet.errors.add(key, 'and snippet files cannot be used together') if params.key?(key) end else - snippet.errors.add(:snippet_files, 'have invalid data') + snippet.errors.add(:snippet_actions, 'have invalid data') end snippet_error_response(snippet, 403) @@ -73,11 +75,15 @@ module Snippets end def files_to_commit(snippet) - snippet_files.to_commit_actions.presence || build_actions_from_params(snippet) + snippet_actions.to_commit_actions.presence || build_actions_from_params(snippet) end def build_actions_from_params(snippet) raise NotImplementedError end + + def restricted_files_actions + nil + end end end diff --git a/app/services/snippets/create_service.rb b/app/services/snippets/create_service.rb index 7b477621da3..dab47de8a36 100644 --- a/app/services/snippets/create_service.rb +++ b/app/services/snippets/create_service.rb @@ -37,13 +37,13 @@ module Snippets end end - # If the snippet_files param is present + # If the snippet_actions param is present # we need to fill content and file_name from # the model def create_params - return params if snippet_files.empty? + return params if snippet_actions.empty? - params.merge(content: snippet_files[0].content, file_name: snippet_files[0].file_path) + params.merge(content: snippet_actions[0].content, file_name: snippet_actions[0].file_path) end def save_and_commit @@ -100,5 +100,9 @@ module Snippets def build_actions_from_params(_snippet) [{ file_path: params[:file_name], content: params[:content] }] end + + def restricted_files_actions + :create + end end end diff --git a/app/services/snippets/update_service.rb b/app/services/snippets/update_service.rb index 6cdc2c374da..00146389e22 100644 --- a/app/services/snippets/update_service.rb +++ b/app/services/snippets/update_service.rb @@ -37,8 +37,9 @@ module Snippets # is implemented. # Once we can perform different operations through this service # we won't need to keep track of the `content` and `file_name` fields - if snippet_files.any? - params.merge!(content: snippet_files[0].content, file_name: snippet_files[0].file_path) + if snippet_actions.any? + params[:content] = snippet_actions[0].content if snippet_actions[0].content + params[:file_name] = snippet_actions[0].file_path end snippet.assign_attributes(params) @@ -108,7 +109,7 @@ module Snippets end def committable_attributes? - (params.stringify_keys.keys & COMMITTABLE_ATTRIBUTES).present? || snippet_files.any? + (params.stringify_keys.keys & COMMITTABLE_ATTRIBUTES).present? || snippet_actions.any? end def build_actions_from_params(snippet) diff --git a/app/services/snippets/update_statistics_service.rb b/app/services/snippets/update_statistics_service.rb new file mode 100644 index 00000000000..295cb963ccc --- /dev/null +++ b/app/services/snippets/update_statistics_service.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Snippets + class UpdateStatisticsService + attr_reader :snippet + + def initialize(snippet) + @snippet = snippet + end + + def execute + unless snippet.repository_exists? + return ServiceResponse.error(message: 'Invalid snippet repository', http_status: 400) + end + + snippet.repository.expire_statistics_caches + statistics.refresh! + + ServiceResponse.success(message: 'Snippet statistics successfully updated.') + end + + private + + def statistics + @statistics ||= snippet.statistics || snippet.build_statistics + end + end +end diff --git a/app/services/spam/spam_verdict_service.rb b/app/services/spam/spam_verdict_service.rb index 68f1135ae28..7de3bad607a 100644 --- a/app/services/spam/spam_verdict_service.rb +++ b/app/services/spam/spam_verdict_service.rb @@ -14,7 +14,7 @@ module Spam end def execute - external_spam_check_result = spam_verdict + external_spam_check_result = external_verdict akismet_result = akismet_verdict # filter out anything we don't recognise, including nils. @@ -38,7 +38,7 @@ module Spam end end - def spam_verdict + def external_verdict return unless Gitlab::CurrentSettings.spam_check_endpoint_enabled return if endpoint_url.blank? @@ -50,17 +50,14 @@ module Spam # @TODO metrics/logging # Expecting: # error: (string or nil) - # result: (string or nil) - verdict = json_result[:verdict] - return unless SUPPORTED_VERDICTS.include?(verdict) - + # verdict: (string or nil) # @TODO log if json_result[:error] json_result[:verdict] rescue *Gitlab::HTTP::HTTP_ERRORS => e # @TODO: log error via try_post https://gitlab.com/gitlab-org/gitlab/-/issues/219223 Gitlab::ErrorTracking.log_exception(e) - return + nil rescue # @TODO log ALLOW diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index 6bf04c55415..db5693960b2 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -273,6 +273,38 @@ module SystemNoteService ::SystemNotes::DesignManagementService.new(noteable: design.issue, project: design.project, author: discussion_note.author).design_discussion_added(discussion_note) end + + # Called when the merge request is approved by user + # + # noteable - Noteable object + # user - User performing approve + # + # Example Note text: + # + # "approved this merge request" + # + # Returns the created Note object + def approve_mr(noteable, user) + merge_requests_service(noteable, noteable.project, user).approve_mr + end + + def unapprove_mr(noteable, user) + merge_requests_service(noteable, noteable.project, user).unapprove_mr + end + + def change_alert_status(alert, author) + ::SystemNotes::AlertManagementService.new(noteable: alert, project: alert.project, author: author).change_alert_status(alert) + end + + def new_alert_issue(alert, issue, author) + ::SystemNotes::AlertManagementService.new(noteable: alert, project: alert.project, author: author).new_alert_issue(alert, issue) + end + + private + + def merge_requests_service(noteable, project, author) + ::SystemNotes::MergeRequestsService.new(noteable: noteable, project: project, author: author) + end end SystemNoteService.prepend_if_ee('EE::SystemNoteService') diff --git a/app/services/system_notes/alert_management_service.rb b/app/services/system_notes/alert_management_service.rb new file mode 100644 index 00000000000..55a6a17bbca --- /dev/null +++ b/app/services/system_notes/alert_management_service.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module SystemNotes + class AlertManagementService < ::SystemNotes::BaseService + # Called when the status of an AlertManagement::Alert has changed + # + # alert - AlertManagement::Alert object. + # + # Example Note text: + # + # "changed the status to Acknowledged" + # + # Returns the created Note object + def change_alert_status(alert) + status = AlertManagement::Alert::STATUSES.key(alert.status).to_s.titleize + body = "changed the status to **#{status}**" + + create_note(NoteSummary.new(noteable, project, author, body, action: 'status')) + end + + # Called when an issue is created based on an AlertManagement::Alert + # + # alert - AlertManagement::Alert object. + # issue - Issue object. + # + # Example Note text: + # + # "created issue #17 for this alert" + # + # Returns the created Note object + def new_alert_issue(alert, issue) + body = "created issue #{issue.to_reference(project)} for this alert" + + create_note(NoteSummary.new(noteable, project, author, body, action: 'alert_issue_added')) + end + end +end diff --git a/app/services/system_notes/issuables_service.rb b/app/services/system_notes/issuables_service.rb index 7d7ee8d829e..76261aa716e 100644 --- a/app/services/system_notes/issuables_service.rb +++ b/app/services/system_notes/issuables_service.rb @@ -228,7 +228,9 @@ module SystemNotes # A state event which results in a synthetic note will be # created by EventCreateService if change event tracking # is enabled. - unless state_change_tracking_enabled? + if state_change_tracking_enabled? + create_resource_state_event(status: status, mentionable_source: source) + else create_note(NoteSummary.new(noteable, project, author, body, action: action)) end end @@ -288,15 +290,23 @@ module SystemNotes end def close_after_error_tracking_resolve - body = _('resolved the corresponding error and closed the issue.') + if state_change_tracking_enabled? + create_resource_state_event(status: 'closed', close_after_error_tracking_resolve: true) + else + body = 'resolved the corresponding error and closed the issue.' - create_note(NoteSummary.new(noteable, project, author, body, action: 'closed')) + create_note(NoteSummary.new(noteable, project, author, body, action: 'closed')) + end end def auto_resolve_prometheus_alert - body = 'automatically closed this issue because the alert resolved.' + if state_change_tracking_enabled? + create_resource_state_event(status: 'closed', close_auto_resolve_prometheus_alert: true) + else + body = 'automatically closed this issue because the alert resolved.' - create_note(NoteSummary.new(noteable, project, author, body, action: 'closed')) + create_note(NoteSummary.new(noteable, project, author, body, action: 'closed')) + end end private @@ -324,6 +334,11 @@ module SystemNotes note_text =~ /\A#{cross_reference_note_prefix}/i end + def create_resource_state_event(params) + ResourceEvents::ChangeStateService.new(resource: noteable, user: author) + .execute(params) + end + def state_change_tracking_enabled? noteable.respond_to?(:resource_state_events) && ::Feature.enabled?(:track_resource_state_change_events, noteable.project) diff --git a/app/services/system_notes/merge_requests_service.rb b/app/services/system_notes/merge_requests_service.rb index baf26245eb9..9b5c9ba20b2 100644 --- a/app/services/system_notes/merge_requests_service.rb +++ b/app/services/system_notes/merge_requests_service.rb @@ -150,7 +150,24 @@ module SystemNotes create_note(summary) end + + # Called when the merge request is approved by user + # + # Example Note text: + # + # "approved this merge request" + # + # Returns the created Note object + def approve_mr + body = "approved this merge request" + + create_note(NoteSummary.new(noteable, project, author, body, action: 'approved')) + end + + def unapprove_mr + body = "unapproved this merge request" + + create_note(NoteSummary.new(noteable, project, author, body, action: 'unapproved')) + end end end - -SystemNotes::MergeRequestsService.prepend_if_ee('::EE::SystemNotes::MergeRequestsService') diff --git a/app/services/tags/destroy_service.rb b/app/services/tags/destroy_service.rb index 3a01192487d..4d1f4043b01 100644 --- a/app/services/tags/destroy_service.rb +++ b/app/services/tags/destroy_service.rb @@ -18,6 +18,8 @@ module Tags .new(project, current_user, tag: tag_name) .execute + unlock_artifacts(tag_name) + success('Tag was removed') else error('Failed to remove tag') @@ -33,5 +35,11 @@ module Tags def success(message) super().merge(message: message) end + + private + + def unlock_artifacts(tag_name) + Ci::RefDeleteUnlockArtifactsWorker.perform_async(project.id, current_user.id, "#{::Gitlab::Git::TAG_REF_PREFIX}#{tag_name}") + end end end diff --git a/app/services/terraform/remote_state_handler.rb b/app/services/terraform/remote_state_handler.rb index d180a3a2432..d2c44d4a265 100644 --- a/app/services/terraform/remote_state_handler.rb +++ b/app/services/terraform/remote_state_handler.rb @@ -5,26 +5,17 @@ module Terraform include Gitlab::OptimisticLocking StateLockedError = Class.new(StandardError) + UnauthorizedError = Class.new(StandardError) - # rubocop: disable CodeReuse/ActiveRecord def find_with_lock - raise ArgumentError unless params[:name].present? - - state = Terraform::State.find_by(project: project, name: params[:name]) - raise ActiveRecord::RecordNotFound.new("Couldn't find state") unless state - - retry_optimistic_lock(state) { |state| yield state } if state && block_given? - state - end - # rubocop: enable CodeReuse/ActiveRecord - - def create_or_find! - raise ArgumentError unless params[:name].present? - - Terraform::State.create_or_find_by(project: project, name: params[:name]) + retrieve_with_lock(find_only: true) do |state| + yield state if block_given? + end end def handle_with_lock + raise UnauthorizedError unless can_modify_state? + retrieve_with_lock do |state| raise StateLockedError unless lock_matches?(state) @@ -36,6 +27,7 @@ module Terraform def lock! raise ArgumentError if params[:lock_id].blank? + raise UnauthorizedError unless can_modify_state? retrieve_with_lock do |state| raise StateLockedError if state.locked? @@ -49,6 +41,8 @@ module Terraform end def unlock! + raise UnauthorizedError unless can_modify_state? + retrieve_with_lock do |state| # force-unlock does not pass ID, so we ignore it if it is missing raise StateLockedError unless params[:lock_id].nil? || lock_matches?(state) @@ -63,8 +57,21 @@ module Terraform private - def retrieve_with_lock - create_or_find!.tap { |state| retry_optimistic_lock(state) { |state| yield state } } + def retrieve_with_lock(find_only: false) + create_or_find!(find_only: find_only).tap { |state| retry_optimistic_lock(state) { |state| yield state } } + end + + def create_or_find!(find_only:) + raise ArgumentError unless params[:name].present? + + find_params = { project: project, name: params[:name] } + + if find_only + Terraform::State.find_by(find_params) || # rubocop: disable CodeReuse/ActiveRecord + raise(ActiveRecord::RecordNotFound.new("Couldn't find state")) + else + Terraform::State.create_or_find_by(find_params) + end end def lock_matches?(state) @@ -73,5 +80,9 @@ module Terraform ActiveSupport::SecurityUtils .secure_compare(state.lock_xid.to_s, params[:lock_id].to_s) end + + def can_modify_state? + current_user.can?(:admin_terraform_state, project) + end end end diff --git a/app/services/todo_service.rb b/app/services/todo_service.rb index e6fb0d3c72e..ec15bdde8d7 100644 --- a/app/services/todo_service.rb +++ b/app/services/todo_service.rb @@ -162,9 +162,9 @@ class TodoService create_assignment_todo(alert, current_user, []) end - # When user marks an issue as todo - def mark_todo(issuable, current_user) - attributes = attributes_for_todo(issuable.project, issuable, current_user, Todo::MARKED) + # When user marks a target as todo + def mark_todo(target, current_user) + attributes = attributes_for_todo(target.project, target, current_user, Todo::MARKED) create_todos(current_user, attributes) end diff --git a/app/services/update_container_registry_info_service.rb b/app/services/update_container_registry_info_service.rb new file mode 100644 index 00000000000..531335839a9 --- /dev/null +++ b/app/services/update_container_registry_info_service.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class UpdateContainerRegistryInfoService + def execute + registry_config = Gitlab.config.registry + return unless registry_config.enabled && registry_config.api_url.presence + + # registry_info will query the /v2 route of the registry API. This route + # requires authentication, but not authorization (the response has no body, + # only headers that show the version of the registry). There might be no + # associated user when running this (e.g. from a rake task or a cron job), + # so we need to generate a valid JWT token with no access permissions to + # authenticate as a trusted client. + token = Auth::ContainerRegistryAuthenticationService.access_token([], []) + client = ContainerRegistry::Client.new(registry_config.api_url, token: token) + info = client.registry_info + + Gitlab::CurrentSettings.update!( + container_registry_vendor: info[:vendor] || '', + container_registry_version: info[:version] || '', + container_registry_features: info[:features] || [] + ) + end +end diff --git a/app/services/users/block_service.rb b/app/services/users/block_service.rb index 9c393832d8f..041db731875 100644 --- a/app/services/users/block_service.rb +++ b/app/services/users/block_service.rb @@ -19,7 +19,7 @@ module Users private def after_block_hook(user) - # overriden by EE module + # overridden by EE module end end end diff --git a/app/services/wiki_pages/base_service.rb b/app/services/wiki_pages/base_service.rb index a0256ea5e69..2967684f7bc 100644 --- a/app/services/wiki_pages/base_service.rb +++ b/app/services/wiki_pages/base_service.rb @@ -44,8 +44,6 @@ module WikiPages end def create_wiki_event(page) - return unless ::Feature.enabled?(:wiki_events) - response = WikiPages::EventCreateService.new(current_user).execute(slug_for_page(page), page, event_action) log_error(response.message) if response.error? diff --git a/app/services/wiki_pages/event_create_service.rb b/app/services/wiki_pages/event_create_service.rb index 18a45d057a9..0453c90d693 100644 --- a/app/services/wiki_pages/event_create_service.rb +++ b/app/services/wiki_pages/event_create_service.rb @@ -10,8 +10,6 @@ module WikiPages end def execute(slug, page, action) - return ServiceResponse.success(message: 'No event created as `wiki_events` feature is disabled') unless ::Feature.enabled?(:wiki_events) - event = Event.transaction do wiki_page_meta = WikiPage::Meta.find_or_create(slug, page) |