diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
commit | 9f46488805e86b1bc341ea1620b866016c2ce5ed (patch) | |
tree | f9748c7e287041e37d6da49e0a29c9511dc34768 /app/services | |
parent | dfc92d081ea0332d69c8aca2f0e745cb48ae5e6d (diff) | |
download | gitlab-ce-9f46488805e86b1bc341ea1620b866016c2ce5ed.tar.gz |
Add latest changes from gitlab-org/gitlab@13-0-stable-ee
Diffstat (limited to 'app/services')
129 files changed, 2527 insertions, 629 deletions
diff --git a/app/services/alert_management/create_alert_issue_service.rb b/app/services/alert_management/create_alert_issue_service.rb new file mode 100644 index 00000000000..0197f29145d --- /dev/null +++ b/app/services/alert_management/create_alert_issue_service.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module AlertManagement + class CreateAlertIssueService + # @param alert [AlertManagement::Alert] + # @param user [User] + def initialize(alert, user) + @alert = alert + @user = user + end + + def execute + return error_no_permissions unless allowed? + return error_issue_already_exists if alert.issue + + result = create_issue(alert, user, alert_payload) + @issue = result[:issue] + + return error(result[:message]) if result[:status] == :error + return error(alert.errors.full_messages.to_sentence) unless update_alert_issue_id + + success + end + + private + + attr_reader :alert, :user, :issue + + delegate :project, to: :alert + + def allowed? + Feature.enabled?(:alert_management_create_alert_issue, project) && + user.can?(:create_issue, project) + end + + def create_issue(alert, user, alert_payload) + ::IncidentManagement::CreateIssueService + .new(project, alert_payload, user) + .execute(skip_settings_check: true) + end + + def alert_payload + if alert.prometheus? + alert.payload + else + Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h) + end + end + + def update_alert_issue_id + alert.update(issue_id: issue.id) + end + + def success + ServiceResponse.success(payload: { issue: issue }) + end + + def error(message) + ServiceResponse.error(payload: { issue: issue }, message: message) + end + + def error_issue_already_exists + error(_('An issue already exists')) + end + + def error_no_permissions + error(_('You have no permissions')) + end + end +end diff --git a/app/services/alert_management/process_prometheus_alert_service.rb b/app/services/alert_management/process_prometheus_alert_service.rb new file mode 100644 index 00000000000..af28f1354b3 --- /dev/null +++ b/app/services/alert_management/process_prometheus_alert_service.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module AlertManagement + class ProcessPrometheusAlertService < BaseService + include Gitlab::Utils::StrongMemoize + + def execute + return bad_request unless parsed_alert.valid? + + process_alert_management_alert + + ServiceResponse.success + end + + private + + delegate :firing?, :resolved?, :gitlab_fingerprint, :ends_at, to: :parsed_alert + + def parsed_alert + strong_memoize(:parsed_alert) do + Gitlab::Alerting::Alert.new(project: project, payload: params) + end + end + + def process_alert_management_alert + process_firing_alert_management_alert if firing? + process_resolved_alert_management_alert if resolved? + end + + def process_firing_alert_management_alert + if am_alert.present? + reset_alert_management_alert_status + else + create_alert_management_alert + end + end + + def reset_alert_management_alert_status + return if am_alert.trigger + + logger.warn( + message: 'Unable to update AlertManagement::Alert status to triggered', + project_id: project.id, + alert_id: am_alert.id + ) + end + + def create_alert_management_alert + am_alert = AlertManagement::Alert.new(am_alert_params.merge(ended_at: nil)) + return if am_alert.save + + logger.warn( + message: 'Unable to create AlertManagement::Alert', + project_id: project.id, + alert_errors: am_alert.errors.messages + ) + end + + def am_alert_params + Gitlab::AlertManagement::AlertParams.from_prometheus_alert(project: project, parsed_alert: parsed_alert) + end + + def process_resolved_alert_management_alert + return if am_alert.blank? + return if am_alert.resolve(ends_at) + + logger.warn( + message: 'Unable to update AlertManagement::Alert status to resolved', + project_id: project.id, + alert_id: am_alert.id + ) + end + + def logger + @logger ||= Gitlab::AppLogger + end + + def am_alert + @am_alert ||= AlertManagement::Alert.for_fingerprint(project, gitlab_fingerprint).first + end + + def bad_request + ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) + end + end +end diff --git a/app/services/alert_management/update_alert_status_service.rb b/app/services/alert_management/update_alert_status_service.rb new file mode 100644 index 00000000000..a7ebddb82e0 --- /dev/null +++ b/app/services/alert_management/update_alert_status_service.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module AlertManagement + class UpdateAlertStatusService + include Gitlab::Utils::StrongMemoize + + # @param alert [AlertManagement::Alert] + # @param user [User] + # @param status [Integer] Must match a value from AlertManagement::Alert::STATUSES + def initialize(alert, user, status) + @alert = alert + @user = user + @status = status + end + + def execute + return error_no_permissions unless allowed? + return error_invalid_status unless status_key + + if alert.update(status_event: status_event) + success + else + error(alert.errors.full_messages.to_sentence) + end + end + + private + + attr_reader :alert, :user, :status + + delegate :project, to: :alert + + def allowed? + user.can?(:update_alert_management_alert, project) + end + + def status_key + strong_memoize(:status_key) do + AlertManagement::Alert::STATUSES.key(status) + end + end + + def status_event + AlertManagement::Alert::STATUS_EVENTS[status_key] + end + + def success + ServiceResponse.success(payload: { alert: alert }) + end + + def error_no_permissions + error(_('You have no permissions')) + end + + def error_invalid_status + error(_('Invalid status')) + end + + def error(message) + ServiceResponse.error(payload: { alert: alert }, message: message) + end + end +end diff --git a/app/services/audit_event_service.rb b/app/services/audit_event_service.rb index d9e40c456aa..fb309aed649 100644 --- a/app/services/audit_event_service.rb +++ b/app/services/audit_event_service.rb @@ -50,8 +50,9 @@ class AuditEventService private def build_author(author) - if author.is_a?(User) - author + case author + when User + author.impersonated? ? Gitlab::Audit::ImpersonatedAuthor.new(author) : author else Gitlab::Audit::UnauthenticatedAuthor.new(name: author) end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 4a699fe3213..44a434f4402 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -52,7 +52,7 @@ module Auth end def self.token_expire_at - Time.now + Gitlab::CurrentSettings.container_registry_token_expire_delay.minutes + Time.current + Gitlab::CurrentSettings.container_registry_token_expire_delay.minutes end private @@ -103,17 +103,19 @@ module Auth return unless requested_project - actions = actions.select do |action| + authorized_actions = actions.select do |action| can_access?(requested_project, action) end - return unless actions.present? + log_if_actions_denied(type, requested_project, actions, authorized_actions) + + return unless authorized_actions.present? # At this point user/build is already authenticated. # - ensure_container_repository!(path, actions) + ensure_container_repository!(path, authorized_actions) - { type: type, name: path.to_s, actions: actions } + { type: type, name: path.to_s, actions: authorized_actions } end ## @@ -222,5 +224,22 @@ module Auth REGISTRY_LOGIN_ABILITIES.include?(ability) end end + + def log_if_actions_denied(type, requested_project, requested_actions, authorized_actions) + return if requested_actions == authorized_actions + + log_info = { + message: "Denied container registry permissions", + scope_type: type, + requested_project_path: requested_project.full_path, + requested_actions: requested_actions, + authorized_actions: authorized_actions, + username: current_user&.username, + user_id: current_user&.id, + project_path: project&.full_path + }.compact + + Gitlab::AuthLogger.warn(log_info) + end end end diff --git a/app/services/authorized_project_update/project_create_service.rb b/app/services/authorized_project_update/project_create_service.rb new file mode 100644 index 00000000000..c17c0a033fe --- /dev/null +++ b/app/services/authorized_project_update/project_create_service.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module AuthorizedProjectUpdate + class ProjectCreateService < BaseService + BATCH_SIZE = 1000 + + def initialize(project) + @project = project + end + + def execute + group = project.group + + unless group + return ServiceResponse.error(message: 'Project does not have a group') + end + + group.members_from_self_and_ancestors_with_effective_access_level + .each_batch(of: BATCH_SIZE, column: :user_id) do |members| + attributes = members.map do |member| + { user_id: member.user_id, project_id: project.id, access_level: member.access_level } + end + + ProjectAuthorization.insert_all(attributes) + end + + ServiceResponse.success + end + + private + + attr_reader :project + end +end diff --git a/app/services/base_container_service.rb b/app/services/base_container_service.rb new file mode 100644 index 00000000000..56e4b8c908c --- /dev/null +++ b/app/services/base_container_service.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Base class, scoped by container (project or group) +class BaseContainerService + include BaseServiceUtility + + attr_reader :container, :current_user, :params + + def initialize(container:, current_user: nil, params: {}) + @container, @current_user, @params = container, current_user, params.dup + end +end diff --git a/app/services/base_service.rb b/app/services/base_service.rb index bc0b968f516..b4c4b6980a8 100644 --- a/app/services/base_service.rb +++ b/app/services/base_service.rb @@ -1,7 +1,16 @@ # frozen_string_literal: true +# This is the original root class for service related classes, +# and due to historical reason takes a project as scope. +# Later separate base classes for different scopes will be created, +# and existing service will use these one by one. +# After all are migrated, we can remove this class. +# +# TODO: New services should consider inheriting from +# BaseContainerService, or create new base class: +# https://gitlab.com/gitlab-org/gitlab/-/issues/216672 class BaseService - include Gitlab::Allowable + include BaseServiceUtility attr_accessor :project, :current_user, :params @@ -9,67 +18,5 @@ class BaseService @project, @current_user, @params = project, user, params.dup end - def notification_service - NotificationService.new - end - - def event_service - EventCreateService.new - end - - def todo_service - TodoService.new - end - - def log_info(message) - Gitlab::AppLogger.info message - end - - def log_error(message) - Gitlab::AppLogger.error message - end - - def system_hook_service - SystemHooksService.new - end - delegate :repository, to: :project - - # Add an error to the specified model for restricted visibility levels - def deny_visibility_level(model, denied_visibility_level = nil) - denied_visibility_level ||= model.visibility_level - - level_name = Gitlab::VisibilityLevel.level_name(denied_visibility_level).downcase - - model.errors.add(:visibility_level, "#{level_name} has been restricted by your GitLab administrator") - end - - def visibility_level - params[:visibility].is_a?(String) ? Gitlab::VisibilityLevel.level_value(params[:visibility]) : params[:visibility_level] - end - - private - - # Return a Hash with an `error` status - # - # message - Error message to include in the Hash - # http_status - Optional HTTP status code override (default: nil) - # pass_back - Additional attributes to be included in the resulting Hash - def error(message, http_status = nil, pass_back: {}) - result = { - message: message, - status: :error - }.reverse_merge(pass_back) - - result[:http_status] = http_status if http_status - result - end - - # Return a Hash with a `success` status - # - # pass_back - Additional attributes to be included in the resulting Hash - def success(pass_back = {}) - pass_back[:status] = :success - pass_back - end end diff --git a/app/services/boards/issues/list_service.rb b/app/services/boards/issues/list_service.rb index 9637eb1b918..e08509b84db 100644 --- a/app/services/boards/issues/list_service.rb +++ b/app/services/boards/issues/list_service.rb @@ -12,7 +12,7 @@ module Boards def execute return fetch_issues.order_closed_date_desc if list&.closed? - fetch_issues.order_by_position_and_priority(with_cte: can_attempt_search_optimization?) + fetch_issues.order_by_position_and_priority(with_cte: params[:search].present?) end # rubocop: disable CodeReuse/ActiveRecord @@ -91,7 +91,7 @@ module Boards end def set_attempt_search_optimizations - return unless can_attempt_search_optimization? + return unless params[:search].present? if board.group_board? params[:attempt_group_search_optimizations] = true @@ -130,11 +130,6 @@ module Boards def board_group board.group_board? ? parent : parent.group end - - def can_attempt_search_optimization? - params[:search].present? && - Feature.enabled?(:board_search_optimization, board_group, default_enabled: true) - end end end end diff --git a/app/services/boards/lists/list_service.rb b/app/services/boards/lists/list_service.rb index c96ea970943..07ce58b6851 100644 --- a/app/services/boards/lists/list_service.rb +++ b/app/services/boards/lists/list_service.rb @@ -3,8 +3,10 @@ module Boards module Lists class ListService < Boards::BaseService - def execute(board) - board.lists.create(list_type: :backlog) unless board.lists.backlog.exists? + def execute(board, create_default_lists: true) + if create_default_lists && !board.lists.backlog.exists? + board.lists.create(list_type: :backlog) + end board.lists.preload_associated_models end diff --git a/app/services/branches/create_service.rb b/app/services/branches/create_service.rb index c8afd97e6bf..958dd5c9965 100644 --- a/app/services/branches/create_service.rb +++ b/app/services/branches/create_service.rb @@ -14,7 +14,7 @@ module Branches if new_branch success(new_branch) else - error("Invalid reference name: #{branch_name}") + error("Invalid reference name: #{ref}") end rescue Gitlab::Git::PreReceiveError => ex error(ex.message) diff --git a/app/services/ci/compare_accessibility_reports_service.rb b/app/services/ci/compare_accessibility_reports_service.rb new file mode 100644 index 00000000000..efb38d39d98 --- /dev/null +++ b/app/services/ci/compare_accessibility_reports_service.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Ci + class CompareAccessibilityReportsService < CompareReportsBaseService + def comparer_class + Gitlab::Ci::Reports::AccessibilityReportsComparer + end + + def serializer_class + AccessibilityReportsComparerSerializer + end + + def get_report(pipeline) + pipeline&.accessibility_reports + end + end +end diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb index 5d7d552dc5a..f0ffe67510b 100644 --- a/app/services/ci/create_job_artifacts_service.rb +++ b/app/services/ci/create_job_artifacts_service.rb @@ -46,6 +46,11 @@ module Ci expire_in: expire_in) end + if Feature.enabled?(:keep_latest_artifact_for_ref, job.project) + artifact.locked = true + artifact_metadata&.locked = true + end + [artifact, artifact_metadata] end @@ -56,6 +61,7 @@ module Ci case artifact.file_type when 'dotenv' then parse_dotenv_artifact(job, artifact) + when 'cluster_applications' then parse_cluster_applications_artifact(job, artifact) else success end end @@ -64,6 +70,7 @@ module Ci Ci::JobArtifact.transaction do artifact.save! artifact_metadata&.save! + unlock_previous_artifacts!(artifact) # NOTE: The `artifacts_expire_at` column is already deprecated and to be removed in the near future. job.update_column(:artifacts_expire_at, artifact.expire_at) @@ -81,6 +88,12 @@ module Ci error(error.message, :bad_request) end + def unlock_previous_artifacts!(artifact) + return unless Feature.enabled?(:keep_latest_artifact_for_ref, artifact.job.project) + + Ci::JobArtifact.for_ref(artifact.job.ref, artifact.project_id).locked.update_all(locked: false) + end + def sha256_matches_existing_artifact?(job, artifact_type, artifacts_file) existing_artifact = job.job_artifacts.find_by_file_type(artifact_type) return false unless existing_artifact @@ -99,5 +112,9 @@ module Ci def parse_dotenv_artifact(job, artifact) Ci::ParseDotenvArtifactService.new(job.project, current_user).execute(artifact) end + + def parse_cluster_applications_artifact(job, artifact) + Clusters::ParseClusterApplicationsArtifactService.new(job, job.user).execute(artifact) + end end end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 347630f865f..922c3556362 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -102,21 +102,12 @@ module Ci # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - # TODO: Introduced by https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/23464 - if Feature.enabled?(:ci_support_interruptible_pipelines, project, default_enabled: true) - project.ci_pipelines - .where(ref: pipeline.ref) - .where.not(id: pipeline.same_family_pipeline_ids) - .where.not(sha: project.commit(pipeline.ref).try(:id)) - .alive_or_scheduled - .with_only_interruptible_builds - else - project.ci_pipelines - .where(ref: pipeline.ref) - .where.not(id: pipeline.same_family_pipeline_ids) - .where.not(sha: project.commit(pipeline.ref).try(:id)) - .created_or_pending - end + project.ci_pipelines + .where(ref: pipeline.ref) + .where.not(id: pipeline.same_family_pipeline_ids) + .where.not(sha: project.commit(pipeline.ref).try(:id)) + .alive_or_scheduled + .with_only_interruptible_builds end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/services/ci/daily_report_result_service.rb b/app/services/ci/daily_build_group_report_result_service.rb index b774a806203..6cdf3c88f8c 100644 --- a/app/services/ci/daily_report_result_service.rb +++ b/app/services/ci/daily_build_group_report_result_service.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true module Ci - class DailyReportResultService + class DailyBuildGroupReportResultService def execute(pipeline) return unless Feature.enabled?(:ci_daily_code_coverage, pipeline.project, default_enabled: true) - DailyReportResult.upsert_reports(coverage_reports(pipeline)) + DailyBuildGroupReportResult.upsert_reports(coverage_reports(pipeline)) end private @@ -14,15 +14,16 @@ module Ci base_attrs = { project_id: pipeline.project_id, ref_path: pipeline.source_ref_path, - param_type: DailyReportResult.param_types[:coverage], date: pipeline.created_at.to_date, last_pipeline_id: pipeline.id } aggregate(pipeline.builds.with_coverage).map do |group_name, group| base_attrs.merge( - title: group_name, - value: average_coverage(group) + group_name: group_name, + data: { + 'coverage' => average_coverage(group) + } ) end end diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb index 7d2f5d33fed..5deb84812ac 100644 --- a/app/services/ci/destroy_expired_job_artifacts_service.rb +++ b/app/services/ci/destroy_expired_job_artifacts_service.rb @@ -28,7 +28,13 @@ module Ci private def destroy_batch - artifacts = Ci::JobArtifact.expired(BATCH_SIZE).to_a + artifact_batch = if Feature.enabled?(:keep_latest_artifact_for_ref) + Ci::JobArtifact.expired(BATCH_SIZE).unlocked + else + Ci::JobArtifact.expired(BATCH_SIZE) + end + + artifacts = artifact_batch.to_a return false if artifacts.empty? diff --git a/app/services/ci/generate_terraform_reports_service.rb b/app/services/ci/generate_terraform_reports_service.rb new file mode 100644 index 00000000000..d768ce777d4 --- /dev/null +++ b/app/services/ci/generate_terraform_reports_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Ci + # TODO: a couple of points with this approach: + # + reuses existing architecture and reactive caching + # - it's not a report comparison and some comparing features must be turned off. + # see CompareReportsBaseService for more notes. + # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224 + class GenerateTerraformReportsService < CompareReportsBaseService + def execute(base_pipeline, head_pipeline) + { + status: :parsed, + key: key(base_pipeline, head_pipeline), + data: head_pipeline.terraform_reports.plans + } + rescue => e + Gitlab::ErrorTracking.track_exception(e, project_id: project.id) + { + status: :error, + key: key(base_pipeline, head_pipeline), + status_reason: _('An error occurred while fetching terraform reports.') + } + end + + def latest?(base_pipeline, head_pipeline, data) + data&.fetch(:key, nil) == key(base_pipeline, head_pipeline) + end + end +end diff --git a/app/services/ci/pipeline_processing/atomic_processing_service.rb b/app/services/ci/pipeline_processing/atomic_processing_service.rb index 2a1bf15b9a3..b01a9d2e3b8 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service.rb @@ -95,7 +95,7 @@ module Ci def processable_status(processable) if processable.scheduling_type_dag? # Processable uses DAG, get status of all dependent needs - @collection.status_for_names(processable.aggregated_needs_names.to_a) + @collection.status_for_names(processable.aggregated_needs_names.to_a, dag: true) else # Processable uses Stages, get status of prior stage @collection.status_for_prior_stage_position(processable.stage_idx.to_i) diff --git a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb index 42e38a5c80f..2228328882d 100644 --- a/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb +++ b/app/services/ci/pipeline_processing/atomic_processing_service/status_collection.rb @@ -32,14 +32,14 @@ module Ci # This methods gets composite status of all processables def status_of_all - status_for_array(all_statuses) + status_for_array(all_statuses, dag: false) end # This methods gets composite status for processables with given names - def status_for_names(names) + def status_for_names(names, dag:) name_statuses = all_statuses_by_name.slice(*names) - status_for_array(name_statuses.values) + status_for_array(name_statuses.values, dag: dag) end # This methods gets composite status for processables before given stage @@ -48,7 +48,7 @@ module Ci stage_statuses = all_statuses_grouped_by_stage_position .select { |stage_position, _| stage_position < position } - status_for_array(stage_statuses.values.flatten) + status_for_array(stage_statuses.values.flatten, dag: false) end end @@ -65,7 +65,7 @@ module Ci strong_memoize("status_for_stage_position_#{current_position}") do stage_statuses = all_statuses_grouped_by_stage_position[current_position].to_a - status_for_array(stage_statuses.flatten) + status_for_array(stage_statuses.flatten, dag: false) end end @@ -76,7 +76,14 @@ module Ci private - def status_for_array(statuses) + def status_for_array(statuses, dag:) + # TODO: This is hack to support + # the same exact behaviour for Atomic and Legacy processing + # that DAG is blocked from executing if dependent is not "complete" + if dag && statuses.any? { |status| HasStatus::COMPLETED_STATUSES.exclude?(status[:status]) } + return 'pending' + end + result = Gitlab::Ci::Status::Composite .new(statuses) .status diff --git a/app/services/ci/pipeline_schedule_service.rb b/app/services/ci/pipeline_schedule_service.rb index 6028643489d..596c3b80bda 100644 --- a/app/services/ci/pipeline_schedule_service.rb +++ b/app/services/ci/pipeline_schedule_service.rb @@ -6,19 +6,7 @@ module Ci # Ensure `next_run_at` is set properly before creating a pipeline. # Otherwise, multiple pipelines could be created in a short interval. schedule.schedule_next_run! - - if Feature.enabled?(:ci_pipeline_schedule_async) - RunPipelineScheduleWorker.perform_async(schedule.id, schedule.owner&.id) - else - begin - RunPipelineScheduleWorker.new.perform(schedule.id, schedule.owner&.id) - ensure - ## - # This is the temporary solution for avoiding the memory bloat. - # See more https://gitlab.com/gitlab-org/gitlab-foss/issues/61955 - GC.start if Feature.enabled?(:ci_pipeline_schedule_force_gc, default_enabled: true) - end - end + RunPipelineScheduleWorker.perform_async(schedule.id, schedule.owner&.id) end end end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index d1efa19eb0d..3f23e81dcdd 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -10,7 +10,6 @@ module Ci def execute(trigger_build_ids = nil, initial_process: false) update_retried - ensure_scheduling_type_for_processables if Feature.enabled?(:ci_atomic_processing, pipeline.project) Ci::PipelineProcessing::AtomicProcessingService @@ -44,17 +43,5 @@ module Ci .update_all(retried: true) if latest_statuses.any? end # rubocop: enable CodeReuse/ActiveRecord - - # Set scheduling type of processables if they were created before scheduling_type - # data was deployed (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22246). - # Given that this service runs multiple times during the pipeline - # life cycle we need to ensure we populate the data once. - # See more: https://gitlab.com/gitlab-org/gitlab/issues/205426 - def ensure_scheduling_type_for_processables - lease = Gitlab::ExclusiveLease.new("set-scheduling-types:#{pipeline.id}", timeout: 1.hour.to_i) - return unless lease.try_obtain - - pipeline.processables.populate_scheduling_type! - end end end diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index fb59797a8df..17b9e56636b 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -85,8 +85,6 @@ module Ci # to make sure that this is properly handled by runner. Result.new(nil, false) rescue => ex - raise ex unless Feature.enabled?(:ci_doom_build, default_enabled: true) - scheduler_failure!(build) track_exception_for_build(ex, build) @@ -203,7 +201,7 @@ module Ci labels[:shard] = shard.gsub(METRICS_SHARD_TAG_PREFIX, '') if shard end - job_queue_duration_seconds.observe(labels, Time.now - job.queued_at) unless job.queued_at.nil? + job_queue_duration_seconds.observe(labels, Time.current - job.queued_at) unless job.queued_at.nil? attempt_counter.increment end diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index a65fe2ecb3a..23507a31c72 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -9,6 +9,8 @@ module Ci resource_group scheduling_type].freeze def execute(build) + build.ensure_scheduling_type! + reprocess!(build).tap do |new_build| build.pipeline.mark_as_processable_after_stage(build.stage_idx) @@ -31,6 +33,9 @@ module Ci end.to_h attributes[:user] = current_user + + # TODO: we can probably remove this logic + # see: https://gitlab.com/gitlab-org/gitlab/-/issues/217930 attributes[:scheduling_type] ||= build.find_legacy_scheduling_type Ci::Build.transaction do diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb index 9bb236ac44c..4229be6c7d7 100644 --- a/app/services/ci/retry_pipeline_service.rb +++ b/app/services/ci/retry_pipeline_service.rb @@ -11,6 +11,8 @@ module Ci needs = Set.new + pipeline.ensure_scheduling_type! + pipeline.retryable_builds.preload_needs.find_each do |build| next unless can?(current_user, :update_build, build) diff --git a/app/services/ci/update_instance_variables_service.rb b/app/services/ci/update_instance_variables_service.rb new file mode 100644 index 00000000000..ee513647d08 --- /dev/null +++ b/app/services/ci/update_instance_variables_service.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +# This class is a simplified version of assign_nested_attributes_for_collection_association from ActiveRecord +# https://github.com/rails/rails/blob/v6.0.2.1/activerecord/lib/active_record/nested_attributes.rb#L466 + +module Ci + class UpdateInstanceVariablesService + UNASSIGNABLE_KEYS = %w(id _destroy).freeze + + def initialize(params) + @params = params[:variables_attributes] + end + + def execute + instantiate_records + persist_records + end + + def errors + @records.to_a.flat_map { |r| r.errors.full_messages } + end + + private + + attr_reader :params + + def existing_records_by_id + @existing_records_by_id ||= Ci::InstanceVariable + .all + .index_by { |var| var.id.to_s } + end + + def instantiate_records + @records = params.map do |attributes| + find_or_initialize_record(attributes).tap do |record| + record.assign_attributes(attributes.except(*UNASSIGNABLE_KEYS)) + record.mark_for_destruction if has_destroy_flag?(attributes) + end + end + end + + def find_or_initialize_record(attributes) + id = attributes[:id].to_s + + if id.blank? + Ci::InstanceVariable.new + else + existing_records_by_id.fetch(id) { raise ActiveRecord::RecordNotFound } + end + end + + def persist_records + Ci::InstanceVariable.transaction do + success = @records.map do |record| + if record.marked_for_destruction? + record.destroy + else + record.save + end + end.all? + + raise ActiveRecord::Rollback unless success + + success + end + end + + def has_destroy_flag?(hash) + Gitlab::Utils.to_boolean(hash['_destroy']) + end + end +end diff --git a/app/services/clusters/applications/base_service.rb b/app/services/clusters/applications/base_service.rb index 86b48b5228d..39a2d6bf758 100644 --- a/app/services/clusters/applications/base_service.rb +++ b/app/services/clusters/applications/base_service.rb @@ -5,6 +5,8 @@ module Clusters class BaseService InvalidApplicationError = Class.new(StandardError) + FLUENTD_KNOWN_ATTRS = %i[host protocol port waf_log_enabled cilium_log_enabled].freeze + attr_reader :cluster, :current_user, :params def initialize(cluster, user, params = {}) @@ -35,17 +37,7 @@ module Clusters application.modsecurity_mode = params[:modsecurity_mode] || 0 end - if application.has_attribute?(:host) - application.host = params[:host] - end - - if application.has_attribute?(:protocol) - application.protocol = params[:protocol] - end - - if application.has_attribute?(:port) - application.port = params[:port] - end + apply_fluentd_related_attributes(application) if application.respond_to?(:oauth_application) application.oauth_application = create_oauth_application(application, request) @@ -111,6 +103,12 @@ module Clusters ::Applications::CreateService.new(current_user, oauth_application_params).execute(request) end + + def apply_fluentd_related_attributes(application) + FLUENTD_KNOWN_ATTRS.each do |attr| + application[attr] = params[attr] if application.has_attribute?(attr) + end + end end end end diff --git a/app/services/clusters/applications/check_installation_progress_service.rb b/app/services/clusters/applications/check_installation_progress_service.rb index 7d064abfaa3..249abd3ff9d 100644 --- a/app/services/clusters/applications/check_installation_progress_service.rb +++ b/app/services/clusters/applications/check_installation_progress_service.rb @@ -33,7 +33,7 @@ module Clusters end def timed_out? - Time.now.utc - app.updated_at.utc > ClusterWaitForAppInstallationWorker::TIMEOUT + Time.current.utc - app.updated_at.utc > ClusterWaitForAppInstallationWorker::TIMEOUT end def remove_installation_pod diff --git a/app/services/clusters/applications/check_uninstall_progress_service.rb b/app/services/clusters/applications/check_uninstall_progress_service.rb index fe9c488bdfd..cd213c3ebbf 100644 --- a/app/services/clusters/applications/check_uninstall_progress_service.rb +++ b/app/services/clusters/applications/check_uninstall_progress_service.rb @@ -31,7 +31,7 @@ module Clusters end def timed_out? - Time.now.utc - app.updated_at.utc > WaitForUninstallAppWorker::TIMEOUT + Time.current.utc - app.updated_at.utc > WaitForUninstallAppWorker::TIMEOUT end def remove_uninstallation_pod diff --git a/app/services/clusters/applications/check_upgrade_progress_service.rb b/app/services/clusters/applications/check_upgrade_progress_service.rb index 8502ea69f27..bc161218618 100644 --- a/app/services/clusters/applications/check_upgrade_progress_service.rb +++ b/app/services/clusters/applications/check_upgrade_progress_service.rb @@ -46,7 +46,7 @@ module Clusters end def timed_out? - Time.now.utc - app.updated_at.to_time.utc > ::ClusterWaitForAppUpdateWorker::TIMEOUT + Time.current.utc - app.updated_at.to_time.utc > ::ClusterWaitForAppUpdateWorker::TIMEOUT end def remove_pod diff --git a/app/services/clusters/applications/ingress_modsecurity_usage_service.rb b/app/services/clusters/applications/ingress_modsecurity_usage_service.rb deleted file mode 100644 index 4aac8bb3cbd..00000000000 --- a/app/services/clusters/applications/ingress_modsecurity_usage_service.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable CodeReuse/ActiveRecord -module Clusters - module Applications - ## - # This service measures usage of the Modsecurity Web Application Firewall across the entire - # instance's deployed environments. - # - # The default configuration is`AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE=DetectionOnly` so we - # measure non-default values via definition of either ci_variables or ci_pipeline_variables. - # Since both these values are encrypted, we must decrypt and count them in memory. - # - # NOTE: this service is an approximation as it does not yet take into account `environment_scope` or `ci_group_variables`. - ## - class IngressModsecurityUsageService - ADO_MODSEC_KEY = "AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE" - - def initialize(blocking_count: 0, disabled_count: 0) - @blocking_count = blocking_count - @disabled_count = disabled_count - end - - def execute - conditions = -> { merge(::Environment.available).merge(::Deployment.success).where(key: ADO_MODSEC_KEY) } - - ci_pipeline_var_enabled = - ::Ci::PipelineVariable - .joins(pipeline: { environments: :last_visible_deployment }) - .merge(conditions) - .order('deployments.environment_id, deployments.id DESC') - - ci_var_enabled = - ::Ci::Variable - .joins(project: { environments: :last_visible_deployment }) - .merge(conditions) - .merge( - # Give priority to pipeline variables by excluding from dataset - ::Ci::Variable.joins(project: :environments).where.not( - environments: { id: ci_pipeline_var_enabled.select('DISTINCT ON (deployments.environment_id) deployments.environment_id') } - ) - ).select('DISTINCT ON (deployments.environment_id) ci_variables.*') - - sum_modsec_config_counts( - ci_pipeline_var_enabled.select('DISTINCT ON (deployments.environment_id) ci_pipeline_variables.*') - ) - sum_modsec_config_counts(ci_var_enabled) - - { - ingress_modsecurity_blocking: @blocking_count, - ingress_modsecurity_disabled: @disabled_count - } - end - - private - - # These are encrypted so we must decrypt and count in memory - def sum_modsec_config_counts(dataset) - dataset.each do |var| - case var.value - when "On" then @blocking_count += 1 - when "Off" then @disabled_count += 1 - # `else` could be default or any unsupported user input - end - end - end - end - end -end diff --git a/app/services/clusters/applications/schedule_update_service.rb b/app/services/clusters/applications/schedule_update_service.rb index b7639c771a8..41718df9a98 100644 --- a/app/services/clusters/applications/schedule_update_service.rb +++ b/app/services/clusters/applications/schedule_update_service.rb @@ -16,9 +16,9 @@ module Clusters return unless application if recently_scheduled? - worker_class.perform_in(BACKOFF_DELAY, application.name, application.id, project.id, Time.now) + worker_class.perform_in(BACKOFF_DELAY, application.name, application.id, project.id, Time.current) else - worker_class.perform_async(application.name, application.id, project.id, Time.now) + worker_class.perform_async(application.name, application.id, project.id, Time.current) end end @@ -31,7 +31,7 @@ module Clusters def recently_scheduled? return false unless application.last_update_started_at - application.last_update_started_at.utc >= Time.now.utc - BACKOFF_DELAY + application.last_update_started_at.utc >= Time.current.utc - BACKOFF_DELAY end end end diff --git a/app/services/clusters/gcp/verify_provision_status_service.rb b/app/services/clusters/gcp/verify_provision_status_service.rb index b24246f5c4b..ddb2832aae6 100644 --- a/app/services/clusters/gcp/verify_provision_status_service.rb +++ b/app/services/clusters/gcp/verify_provision_status_service.rb @@ -35,7 +35,7 @@ module Clusters end def elapsed_time_from_creation(operation) - Time.now.utc - operation.start_time.to_time.utc + Time.current.utc - operation.start_time.to_time.utc end def finalize_creation diff --git a/app/services/clusters/kubernetes/configure_istio_ingress_service.rb b/app/services/clusters/kubernetes/configure_istio_ingress_service.rb index a81014d99ff..53c3c686f07 100644 --- a/app/services/clusters/kubernetes/configure_istio_ingress_service.rb +++ b/app/services/clusters/kubernetes/configure_istio_ingress_service.rb @@ -54,8 +54,8 @@ module Clusters cert = OpenSSL::X509::Certificate.new cert.version = 2 cert.serial = 0 - cert.not_before = Time.now - cert.not_after = Time.now + 1000.years + cert.not_before = Time.current + cert.not_after = Time.current + 1000.years cert.public_key = key.public_key cert.subject = name diff --git a/app/services/clusters/management/create_project_service.rb b/app/services/clusters/management/create_project_service.rb index 0a33582be98..5a0176edd12 100644 --- a/app/services/clusters/management/create_project_service.rb +++ b/app/services/clusters/management/create_project_service.rb @@ -15,11 +15,8 @@ module Clusters def execute return unless management_project_required? - ActiveRecord::Base.transaction do - project = create_management_project! - - update_cluster!(project) - end + project = create_management_project! + update_cluster!(project) end private diff --git a/app/services/clusters/parse_cluster_applications_artifact_service.rb b/app/services/clusters/parse_cluster_applications_artifact_service.rb new file mode 100644 index 00000000000..b8e1c80cfe7 --- /dev/null +++ b/app/services/clusters/parse_cluster_applications_artifact_service.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Clusters + class ParseClusterApplicationsArtifactService < ::BaseService + include Gitlab::Utils::StrongMemoize + + MAX_ACCEPTABLE_ARTIFACT_SIZE = 5.kilobytes + RELEASE_NAMES = %w[prometheus].freeze + + def initialize(job, current_user) + @job = job + + super(job.project, current_user) + end + + def execute(artifact) + return success unless Feature.enabled?(:cluster_applications_artifact, project) + + raise ArgumentError, 'Artifact is not cluster_applications file type' unless artifact&.cluster_applications? + + unless artifact.file.size < MAX_ACCEPTABLE_ARTIFACT_SIZE + return error(too_big_error_message, :bad_request) + end + + unless cluster + return error(s_('ClusterIntegration|No deployment cluster found for this job')) + end + + parse!(artifact) + + success + rescue Gitlab::Kubernetes::Helm::Parsers::ListV2::ParserError, ActiveRecord::RecordInvalid => error + Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id) + error(error.message, :bad_request) + end + + private + + attr_reader :job + + def cluster + strong_memoize(:cluster) do + deployment_cluster = job.deployment&.cluster + + deployment_cluster if Ability.allowed?(current_user, :admin_cluster, deployment_cluster) + end + end + + def parse!(artifact) + releases = [] + + artifact.each_blob do |blob| + releases.concat(Gitlab::Kubernetes::Helm::Parsers::ListV2.new(blob).releases) + end + + update_cluster_application_statuses!(releases) + end + + def update_cluster_application_statuses!(releases) + release_by_name = releases.index_by { |release| release['Name'] } + + Clusters::Cluster.transaction do + RELEASE_NAMES.each do |release_name| + application = find_or_build_application(release_name) + + release = release_by_name[release_name] + + if release + case release['Status'] + when 'DEPLOYED' + application.make_externally_installed! + when 'FAILED' + application.make_errored!(s_('ClusterIntegration|Helm release failed to install')) + end + else + # missing, so by definition, we consider this uninstalled + application.make_externally_uninstalled! if application.persisted? + end + end + end + end + + def find_or_build_application(application_name) + application_class = Clusters::Cluster::APPLICATIONS[application_name] + + cluster.find_or_build_application(application_class) + end + + def too_big_error_message + human_size = ActiveSupport::NumberHelper.number_to_human_size(MAX_ACCEPTABLE_ARTIFACT_SIZE) + + s_('ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}') % { human_size: human_size } + end + end +end diff --git a/app/services/concerns/base_service_utility.rb b/app/services/concerns/base_service_utility.rb new file mode 100644 index 00000000000..70b223a0289 --- /dev/null +++ b/app/services/concerns/base_service_utility.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module BaseServiceUtility + extend ActiveSupport::Concern + include Gitlab::Allowable + + ### Convenience service methods + + def notification_service + NotificationService.new + end + + def event_service + EventCreateService.new + end + + def todo_service + TodoService.new + end + + def system_hook_service + SystemHooksService.new + end + + # Logging + + def log_info(message) + Gitlab::AppLogger.info message + end + + def log_error(message) + Gitlab::AppLogger.error message + end + + # Add an error to the specified model for restricted visibility levels + def deny_visibility_level(model, denied_visibility_level = nil) + denied_visibility_level ||= model.visibility_level + + level_name = Gitlab::VisibilityLevel.level_name(denied_visibility_level).downcase + + model.errors.add(:visibility_level, "#{level_name} has been restricted by your GitLab administrator") + end + + def visibility_level + params[:visibility].is_a?(String) ? Gitlab::VisibilityLevel.level_value(params[:visibility]) : params[:visibility_level] + end + + private + + # Return a Hash with an `error` status + # + # message - Error message to include in the Hash + # http_status - Optional HTTP status code override (default: nil) + # pass_back - Additional attributes to be included in the resulting Hash + def error(message, http_status = nil, pass_back: {}) + result = { + message: message, + status: :error + }.reverse_merge(pass_back) + + result[:http_status] = http_status if http_status + result + end + + # Return a Hash with a `success` status + # + # pass_back - Additional attributes to be included in the resulting Hash + def success(pass_back = {}) + pass_back[:status] = :success + pass_back + end +end diff --git a/app/services/concerns/git/logger.rb b/app/services/concerns/git/logger.rb deleted file mode 100644 index 7c036212e66..00000000000 --- a/app/services/concerns/git/logger.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Git - module Logger - def log_error(message, save_message_on_model: false) - Gitlab::GitLogger.error("#{self.class.name} error (#{merge_request.to_reference(full: true)}): #{message}") - merge_request.update(merge_error: message) if save_message_on_model - end - end -end diff --git a/app/services/concerns/measurable.rb b/app/services/concerns/measurable.rb new file mode 100644 index 00000000000..5a74f15506e --- /dev/null +++ b/app/services/concerns/measurable.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +# In order to measure and log execution of our service, we just need to 'prepend Measurable' module +# Example: +# ``` +# class DummyService +# prepend Measurable +# +# def execute +# # ... +# end +# end + +# DummyService.prepend(Measurable) +# ``` +# +# In case when we are prepending a module from the `EE` namespace with EE features +# we need to prepend Measurable after prepending `EE` module. +# This way Measurable will be at the bottom of the ancestor chain, +# in order to measure execution of `EE` features as well +# ``` +# class DummyService +# def execute +# # ... +# end +# end +# +# DummyService.prepend_if_ee('EE::DummyService') +# DummyService.prepend(Measurable) +# ``` +# +module Measurable + extend ::Gitlab::Utils::Override + + override :execute + def execute(*args) + measuring? ? ::Gitlab::Utils::Measuring.new(base_log_data).with_measuring { super(*args) } : super(*args) + end + + protected + + # You can set extra attributes for performance measurement log. + def extra_attributes_for_measurement + defined?(super) ? super : {} + end + + private + + def measuring? + Feature.enabled?("gitlab_service_measuring_#{service_class}") + end + + # These attributes are always present in log. + def base_log_data + extra_attributes_for_measurement.merge({ class: self.class.name }) + end + + def service_class + self.class.name.underscore.tr('/', '_') + end +end diff --git a/app/services/concerns/spam_check_methods.rb b/app/services/concerns/spam_check_methods.rb index 695bdf92b49..53e9e001463 100644 --- a/app/services/concerns/spam_check_methods.rb +++ b/app/services/concerns/spam_check_methods.rb @@ -23,14 +23,14 @@ module SpamCheckMethods # attribute values. # rubocop:disable Gitlab/ModuleWithInstanceVariables def spam_check(spammable, user) - Spam::SpamCheckService.new( + Spam::SpamActionService.new( spammable: spammable, request: @request ).execute( api: @api, recaptcha_verified: @recaptcha_verified, spam_log_id: @spam_log_id, - user_id: user.id) + user: user) end # rubocop:enable Gitlab/ModuleWithInstanceVariables end diff --git a/app/services/deployments/older_deployments_drop_service.rb b/app/services/deployments/older_deployments_drop_service.rb index 122f8ac89ed..e765d2484ea 100644 --- a/app/services/deployments/older_deployments_drop_service.rb +++ b/app/services/deployments/older_deployments_drop_service.rb @@ -12,7 +12,9 @@ module Deployments return unless @deployment&.running? older_deployments.find_each do |older_deployment| - older_deployment.deployable&.drop!(:forward_deployment_failure) + Gitlab::OptimisticLocking.retry_lock(older_deployment.deployable) do |deployable| + deployable.drop(:forward_deployment_failure) + end rescue => e Gitlab::ErrorTracking.track_exception(e, subject_id: @deployment.id, deployment_id: older_deployment.id) end diff --git a/app/services/design_management/delete_designs_service.rb b/app/services/design_management/delete_designs_service.rb new file mode 100644 index 00000000000..e69f07db5bf --- /dev/null +++ b/app/services/design_management/delete_designs_service.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module DesignManagement + class DeleteDesignsService < DesignService + include RunsDesignActions + include OnSuccessCallbacks + + def initialize(project, user, params = {}) + super + + @designs = params.fetch(:designs) + end + + def execute + return error('Forbidden!') unless can_delete_designs? + + version = delete_designs! + + success(version: version) + end + + def commit_message + n = designs.size + + <<~MSG + Removed #{n} #{'designs'.pluralize(n)} + + #{formatted_file_list} + MSG + end + + private + + attr_reader :designs + + def delete_designs! + DesignManagement::Version.with_lock(project.id, repository) do + run_actions(build_actions) + end + end + + def can_delete_designs? + Ability.allowed?(current_user, :destroy_design, issue) + end + + def build_actions + designs.map { |d| design_action(d) } + end + + def design_action(design) + on_success { counter.count(:delete) } + + DesignManagement::DesignAction.new(design, :delete) + end + + def counter + ::Gitlab::UsageDataCounters::DesignsCounter + end + + def formatted_file_list + designs.map { |design| "- #{design.full_path}" }.join("\n") + end + end +end + +DesignManagement::DeleteDesignsService.prepend_if_ee('EE::DesignManagement::DeleteDesignsService') diff --git a/app/services/design_management/design_service.rb b/app/services/design_management/design_service.rb new file mode 100644 index 00000000000..54e53609646 --- /dev/null +++ b/app/services/design_management/design_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module DesignManagement + class DesignService < ::BaseService + def initialize(project, user, params = {}) + super + + @issue = params.fetch(:issue) + end + + # Accessors common to all subclasses: + + attr_reader :issue + + def target_branch + repository.root_ref || "master" + end + + def collection + issue.design_collection + end + + def repository + collection.repository + end + + def project + issue.project + end + end +end diff --git a/app/services/design_management/design_user_notes_count_service.rb b/app/services/design_management/design_user_notes_count_service.rb new file mode 100644 index 00000000000..e49914ea6d3 --- /dev/null +++ b/app/services/design_management/design_user_notes_count_service.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module DesignManagement + # Service class for counting and caching the number of unresolved + # notes of a Design + class DesignUserNotesCountService < ::BaseCountService + # The version of the cache format. This should be bumped whenever the + # underlying logic changes. This removes the need for explicitly flushing + # all caches. + VERSION = 1 + + def initialize(design) + @design = design + end + + def relation_for_count + design.notes.user + end + + def raw? + # Since we're storing simple integers we don't need all of the + # additional Marshal data Rails includes by default. + true + end + + def cache_key + ['designs', 'notes_count', VERSION, design.id] + end + + private + + attr_reader :design + end +end diff --git a/app/services/design_management/generate_image_versions_service.rb b/app/services/design_management/generate_image_versions_service.rb new file mode 100644 index 00000000000..213aac164ff --- /dev/null +++ b/app/services/design_management/generate_image_versions_service.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module DesignManagement + # This service generates smaller image versions for `DesignManagement::Design` + # records within a given `DesignManagement::Version`. + class GenerateImageVersionsService < DesignService + # We limit processing to only designs with file sizes that don't + # exceed `MAX_DESIGN_SIZE`. + # + # Note, we may be able to remove checking this limit, if when we come to + # implement a file size limit for designs, there are no designs that + # exceed 40MB on GitLab.com + # + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22860#note_281780387 + MAX_DESIGN_SIZE = 40.megabytes.freeze + + def initialize(version) + super(version.project, version.author, issue: version.issue) + + @version = version + end + + def execute + # rubocop: disable CodeReuse/ActiveRecord + version.actions.includes(:design).each do |action| + generate_image(action) + end + # rubocop: enable CodeReuse/ActiveRecord + + success(version: version) + end + + private + + attr_reader :version + + def generate_image(action) + raw_file = get_raw_file(action) + + unless raw_file + log_error("No design file found for Action: #{action.id}") + return + end + + # Skip attempting to process images that would be rejected by CarrierWave. + return unless DesignManagement::DesignV432x230Uploader::MIME_TYPE_WHITELIST.include?(raw_file.content_type) + + # Store and process the file + action.image_v432x230.store!(raw_file) + action.save! + rescue CarrierWave::UploadError => e + Gitlab::ErrorTracking.track_exception(e, project_id: project.id, design_id: action.design_id, version_id: action.version_id) + log_error(e.message) + end + + # Returns the `CarrierWave::SanitizedFile` of the original design file + def get_raw_file(action) + raw_files_by_path[action.design.full_path] + end + + # Returns the `Carrierwave:SanitizedFile` instances for all of the original + # design files, mapping to { design.filename => `Carrierwave::SanitizedFile` }. + # + # As design files are stored in Git LFS, the only way to retrieve their original + # files is to first fetch the LFS pointer file data from the Git design repository. + # The LFS pointer file data contains an "OID" that lets us retrieve `LfsObject` + # records, which have an Uploader (`LfsObjectUploader`) for the original design file. + def raw_files_by_path + @raw_files_by_path ||= begin + LfsObject.for_oids(blobs_by_oid.keys).each_with_object({}) do |lfs_object, h| + blob = blobs_by_oid[lfs_object.oid] + file = lfs_object.file.file + # The `CarrierWave::SanitizedFile` is loaded without knowing the `content_type` + # of the file, due to the file not having an extension. + # + # Set the content_type from the `Blob`. + file.content_type = blob.content_type + h[blob.path] = file + end + end + end + + # Returns the `Blob`s that correspond to the design files in the repository. + # + # All design `Blob`s are LFS Pointer files, and are therefore small amounts + # of data to load. + # + # `Blob`s whose size are above a certain threshold: `MAX_DESIGN_SIZE` + # are filtered out. + def blobs_by_oid + @blobs ||= begin + items = version.designs.map { |design| [version.sha, design.full_path] } + blobs = repository.blobs_at(items) + blobs.reject! { |blob| blob.lfs_size > MAX_DESIGN_SIZE } + blobs.index_by(&:lfs_oid) + end + end + end +end diff --git a/app/services/design_management/on_success_callbacks.rb b/app/services/design_management/on_success_callbacks.rb new file mode 100644 index 00000000000..be55890a02d --- /dev/null +++ b/app/services/design_management/on_success_callbacks.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module DesignManagement + module OnSuccessCallbacks + def on_success(&block) + success_callbacks.push(block) + end + + def success(*_) + while cb = success_callbacks.pop + cb.call + end + + super + end + + private + + def success_callbacks + @success_callbacks ||= [] + end + end +end diff --git a/app/services/design_management/runs_design_actions.rb b/app/services/design_management/runs_design_actions.rb new file mode 100644 index 00000000000..4bd6bb45658 --- /dev/null +++ b/app/services/design_management/runs_design_actions.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module DesignManagement + module RunsDesignActions + NoActions = Class.new(StandardError) + + # this concern requires the following methods to be implemented: + # current_user, target_branch, repository, commit_message + # + # Before calling `run_actions`, you should ensure the repository exists, by + # calling `repository.create_if_not_exists`. + # + # @raise [NoActions] if actions are empty + def run_actions(actions) + raise NoActions if actions.empty? + + sha = repository.multi_action(current_user, + branch_name: target_branch, + message: commit_message, + actions: actions.map(&:gitaly_action)) + + ::DesignManagement::Version + .create_for_designs(actions, sha, current_user) + .tap { |version| post_process(version) } + end + + private + + def post_process(version) + version.run_after_commit_or_now do + ::DesignManagement::NewVersionWorker.perform_async(id) + end + end + end +end diff --git a/app/services/design_management/save_designs_service.rb b/app/services/design_management/save_designs_service.rb new file mode 100644 index 00000000000..a09c19bc885 --- /dev/null +++ b/app/services/design_management/save_designs_service.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module DesignManagement + class SaveDesignsService < DesignService + include RunsDesignActions + include OnSuccessCallbacks + + MAX_FILES = 10 + + def initialize(project, user, params = {}) + super + + @files = params.fetch(:files) + end + + def execute + return error("Not allowed!") unless can_create_designs? + return error("Only #{MAX_FILES} files are allowed simultaneously") if files.size > MAX_FILES + + uploaded_designs, version = upload_designs! + skipped_designs = designs - uploaded_designs + + success({ designs: uploaded_designs, version: version, skipped_designs: skipped_designs }) + rescue ::ActiveRecord::RecordInvalid => e + error(e.message) + end + + private + + attr_reader :files + + def upload_designs! + ::DesignManagement::Version.with_lock(project.id, repository) do + actions = build_actions + + [actions.map(&:design), actions.presence && run_actions(actions)] + end + end + + # Returns `Design` instances that correspond with `files`. + # New `Design`s will be created where a file name does not match + # an existing `Design` + def designs + @designs ||= files.map do |file| + collection.find_or_create_design!(filename: file.original_filename) + end + end + + def build_actions + files.zip(designs).flat_map do |(file, design)| + Array.wrap(build_design_action(file, design)) + end + end + + def build_design_action(file, design) + content = file_content(file, design.full_path) + return if design_unchanged?(design, content) + + action = new_file?(design) ? :create : :update + on_success { ::Gitlab::UsageDataCounters::DesignsCounter.count(action) } + + DesignManagement::DesignAction.new(design, action, content) + end + + # Returns true if the design file is the same as its latest version + def design_unchanged?(design, content) + content == existing_blobs[design]&.data + end + + def commit_message + <<~MSG + Updated #{files.size} #{'designs'.pluralize(files.size)} + + #{formatted_file_list} + MSG + end + + def formatted_file_list + filenames.map { |name| "- #{name}" }.join("\n") + end + + def filenames + @filenames ||= files.map(&:original_filename) + end + + def can_create_designs? + Ability.allowed?(current_user, :create_design, issue) + end + + def new_file?(design) + !existing_blobs[design] + end + + def file_content(file, full_path) + transformer = ::Lfs::FileTransformer.new(project, repository, target_branch) + transformer.new_file(full_path, file.to_io).content + end + + # Returns the latest blobs for the designs as a Hash of `{ Design => Blob }` + def existing_blobs + @existing_blobs ||= begin + items = designs.map { |d| ['HEAD', d.full_path] } + + repository.blobs_at(items).each_with_object({}) do |blob, h| + design = designs.find { |d| d.full_path == blob.path } + + h[design] = blob + end + end + end + end +end + +DesignManagement::SaveDesignsService.prepend_if_ee('EE::DesignManagement::SaveDesignsService') diff --git a/app/services/emails/base_service.rb b/app/services/emails/base_service.rb index 99324638300..c94505b2068 100644 --- a/app/services/emails/base_service.rb +++ b/app/services/emails/base_service.rb @@ -11,3 +11,5 @@ module Emails end end end + +Emails::BaseService.prepend_if_ee('::EE::Emails::BaseService') diff --git a/app/services/event_create_service.rb b/app/services/event_create_service.rb index 0b044e1679a..522f36cda46 100644 --- a/app/services/event_create_service.rb +++ b/app/services/event_create_service.rb @@ -85,18 +85,40 @@ class EventCreateService # Create a new wiki page event # # @param [WikiPage::Meta] wiki_page_meta The event target - # @param [User] current_user The event author + # @param [User] author The event author # @param [Integer] action One of the Event::WIKI_ACTIONS - def wiki_event(wiki_page_meta, current_user, action) + # + # @return a tuple of event and either :found or :created + def wiki_event(wiki_page_meta, author, action) return unless Feature.enabled?(:wiki_events) raise IllegalActionError, action unless Event::WIKI_ACTIONS.include?(action) - create_record_event(wiki_page_meta, current_user, action) + if duplicate = existing_wiki_event(wiki_page_meta, action) + return duplicate + end + + event = create_record_event(wiki_page_meta, author, action) + # Ensure that the event is linked in time to the metadata, for non-deletes + unless action == Event::DESTROYED + time_stamp = wiki_page_meta.updated_at + event.update_columns(updated_at: time_stamp, created_at: time_stamp) + end + + event end private + def existing_wiki_event(wiki_page_meta, action) + if action == Event::DESTROYED + most_recent = Event.for_wiki_meta(wiki_page_meta).recent.first + return most_recent if most_recent.present? && most_recent.action == action + else + Event.for_wiki_meta(wiki_page_meta).created_at(wiki_page_meta.updated_at).first + end + end + def create_record_event(record, current_user, status) create_event(record.resource_parent, current_user, status, target_id: record.id, target_type: record.class.name) end diff --git a/app/services/git/branch_hooks_service.rb b/app/services/git/branch_hooks_service.rb index e1cc1f8c834..92e7702727c 100644 --- a/app/services/git/branch_hooks_service.rb +++ b/app/services/git/branch_hooks_service.rb @@ -112,7 +112,7 @@ module Git end def enqueue_update_signatures - unsigned = unsigned_x509_shas(commits) & unsigned_gpg_shas(commits) + unsigned = unsigned_x509_shas(limited_commits) & unsigned_gpg_shas(limited_commits) return if unsigned.empty? signable = Gitlab::Git::Commit.shas_with_signatures(project.repository, unsigned) diff --git a/app/services/git/wiki_push_service.rb b/app/services/git/wiki_push_service.rb index d4267d4a3c5..8bdbc28f3e8 100644 --- a/app/services/git/wiki_push_service.rb +++ b/app/services/git/wiki_push_service.rb @@ -2,8 +2,63 @@ module Git class WikiPushService < ::BaseService + # Maximum number of change events we will process on any single push + MAX_CHANGES = 100 + def execute - # This is used in EE + process_changes + end + + private + + def process_changes + return unless can_process_wiki_events? + + push_changes.take(MAX_CHANGES).each do |change| # rubocop:disable CodeReuse/ActiveRecord + next unless change.page.present? + + response = create_event_for(change) + log_error(response.message) if response.error? + end + end + + def can_process_wiki_events? + Feature.enabled?(:wiki_events) && Feature.enabled?(:wiki_events_on_git_push, project) + end + + def push_changes + default_branch_changes.flat_map do |change| + raw_changes(change).map { |raw| Git::WikiPushService::Change.new(wiki, change, raw) } + end + end + + def raw_changes(change) + wiki.repository.raw.raw_changes_between(change[:oldrev], change[:newrev]) + end + + def wiki + project.wiki + end + + def create_event_for(change) + event_service.execute(change.last_known_slug, change.page, change.event_action) + end + + def event_service + @event_service ||= WikiPages::EventCreateService.new(current_user) + end + + def on_default_branch?(change) + project.wiki.default_branch == ::Gitlab::Git.branch_name(change[:ref]) + end + + # See: [Gitlab::GitPostReceive#changes] + def changes + params[:changes] || [] + end + + def default_branch_changes + @default_branch_changes ||= changes.select { |change| on_default_branch?(change) } end end end diff --git a/app/services/git/wiki_push_service/change.rb b/app/services/git/wiki_push_service/change.rb new file mode 100644 index 00000000000..8685850165a --- /dev/null +++ b/app/services/git/wiki_push_service/change.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Git + class WikiPushService + class Change + include Gitlab::Utils::StrongMemoize + + # @param [ProjectWiki] wiki + # @param [Hash] change - must have keys `:oldrev` and `:newrev` + # @param [Gitlab::Git::RawDiffChange] raw_change + def initialize(project_wiki, change, raw_change) + @wiki, @raw_change, @change = project_wiki, raw_change, change + end + + def page + strong_memoize(:page) { wiki.find_page(slug, revision) } + end + + # See [Gitlab::Git::RawDiffChange#extract_operation] for the + # definition of the full range of operation values. + def event_action + case raw_change.operation + when :added + Event::CREATED + when :deleted + Event::DESTROYED + else + Event::UPDATED + end + end + + def last_known_slug + strip_extension(raw_change.old_path || raw_change.new_path) + end + + private + + attr_reader :raw_change, :change, :wiki + + def filename + return raw_change.old_path if deleted? + + raw_change.new_path + end + + def slug + strip_extension(filename) + end + + def revision + return change[:oldrev] if deleted? + + change[:newrev] + end + + def deleted? + raw_change.operation == :deleted + end + + def strip_extension(filename) + return unless filename + + File.basename(filename, File.extname(filename)) + end + end + end +end diff --git a/app/services/grafana/proxy_service.rb b/app/services/grafana/proxy_service.rb index 74fcdc750b0..ac4c3cc091c 100644 --- a/app/services/grafana/proxy_service.rb +++ b/app/services/grafana/proxy_service.rb @@ -12,6 +12,7 @@ module Grafana self.reactive_cache_key = ->(service) { service.cache_key } self.reactive_cache_lease_timeout = 30.seconds self.reactive_cache_refresh_interval = 30.seconds + self.reactive_cache_work_type = :external_dependency self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) } attr_accessor :project, :datasource_id, :proxy_path, :query_params diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb index 8cc31200689..eb1b8d4fcc0 100644 --- a/app/services/groups/create_service.rb +++ b/app/services/groups/create_service.rb @@ -38,6 +38,10 @@ module Groups # overridden in EE end + def remove_unallowed_params + params.delete(:default_branch_protection) unless can?(current_user, :create_group_with_default_branch_protection) + end + def create_chat_team? Gitlab.config.mattermost.enabled && @chat_team && group.chat_team.nil? end diff --git a/app/services/groups/import_export/export_service.rb b/app/services/groups/import_export/export_service.rb index f8715b57d6e..0f2e3bb65f9 100644 --- a/app/services/groups/import_export/export_service.rb +++ b/app/services/groups/import_export/export_service.rb @@ -52,11 +52,11 @@ module Groups end def savers - [tree_exporter, file_saver] + [version_saver, tree_exporter, file_saver] end def tree_exporter - Gitlab::ImportExport::Group::LegacyTreeSaver.new( + tree_exporter_class.new( group: @group, current_user: @current_user, shared: @shared, @@ -64,6 +64,18 @@ module Groups ) end + def tree_exporter_class + if ::Feature.enabled?(:group_export_ndjson, @group&.parent, default_enabled: true) + Gitlab::ImportExport::Group::TreeSaver + else + Gitlab::ImportExport::Group::LegacyTreeSaver + end + end + + def version_saver + Gitlab::ImportExport::VersionSaver.new(shared: shared) + end + def file_saver Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared) end @@ -84,6 +96,8 @@ module Groups group_name: @group.name, message: 'Group Import/Export: Export succeeded' ) + + notification_service.group_was_exported(@group, @current_user) end def notify_error @@ -93,6 +107,12 @@ module Groups error: @shared.errors.join(', '), message: 'Group Import/Export: Export failed' ) + + notification_service.group_was_not_exported(@group, @current_user, @shared.errors) + end + + def notification_service + @notification_service ||= NotificationService.new end end end diff --git a/app/services/groups/import_export/import_service.rb b/app/services/groups/import_export/import_service.rb index f62b9d3c8a6..6f692c98c38 100644 --- a/app/services/groups/import_export/import_service.rb +++ b/app/services/groups/import_export/import_service.rb @@ -27,18 +27,34 @@ module Groups private def import_file - @import_file ||= Gitlab::ImportExport::FileImporter.import(importable: @group, - archive_file: nil, - shared: @shared) + @import_file ||= Gitlab::ImportExport::FileImporter.import( + importable: @group, + archive_file: nil, + shared: @shared + ) end def restorer - @restorer ||= Gitlab::ImportExport::Group::LegacyTreeRestorer.new( - user: @current_user, - shared: @shared, - group: @group, - group_hash: nil - ) + @restorer ||= + if ndjson? + Gitlab::ImportExport::Group::TreeRestorer.new( + user: @current_user, + shared: @shared, + group: @group + ) + else + Gitlab::ImportExport::Group::LegacyTreeRestorer.new( + user: @current_user, + shared: @shared, + group: @group, + group_hash: nil + ) + end + end + + def ndjson? + ::Feature.enabled?(:group_import_ndjson, @group&.parent, default_enabled: true) && + File.exist?(File.join(@shared.export_path, 'tree/groups/_all.ndjson')) end def remove_import_file diff --git a/app/services/groups/update_service.rb b/app/services/groups/update_service.rb index 8635b82461b..948540619ae 100644 --- a/app/services/groups/update_service.rb +++ b/app/services/groups/update_service.rb @@ -66,6 +66,7 @@ module Groups # overridden in EE def remove_unallowed_params params.delete(:emails_disabled) unless can?(current_user, :set_emails_disabled, group) + params.delete(:default_branch_protection) unless can?(current_user, :update_default_branch_protection, group) end def valid_share_with_group_lock_change? diff --git a/app/services/incident_management/create_issue_service.rb b/app/services/incident_management/create_issue_service.rb index 43077e03e6d..4b59dc64cec 100644 --- a/app/services/incident_management/create_issue_service.rb +++ b/app/services/incident_management/create_issue_service.rb @@ -13,12 +13,12 @@ module IncidentManagement DESCRIPTION }.freeze - def initialize(project, params) - super(project, User.alert_bot, params) + def initialize(project, params, user = User.alert_bot) + super(project, user, params) end - def execute - return error_with('setting disabled') unless incident_management_setting.create_issue? + def execute(skip_settings_check: false) + return error_with('setting disabled') unless skip_settings_check || incident_management_setting.create_issue? return error_with('invalid alert') unless alert.valid? issue = create_issue diff --git a/app/services/issuable/clone/attributes_rewriter.rb b/app/services/issuable/clone/attributes_rewriter.rb index 55f5629baac..a78e191c85f 100644 --- a/app/services/issuable/clone/attributes_rewriter.rb +++ b/app/services/issuable/clone/attributes_rewriter.rb @@ -20,6 +20,7 @@ module Issuable copy_resource_label_events copy_resource_weight_events copy_resource_milestone_events + copy_resource_state_events end private @@ -47,8 +48,6 @@ module Issuable end def copy_resource_label_events - entity_key = new_entity.class.name.underscore.foreign_key - copy_events(ResourceLabelEvent.table_name, original_entity.resource_label_events) do |event| event.attributes .except('id', 'reference', 'reference_html') @@ -67,22 +66,39 @@ module Issuable end def copy_resource_milestone_events - entity_key = new_entity.class.name.underscore.foreign_key + return unless milestone_events_supported? copy_events(ResourceMilestoneEvent.table_name, original_entity.resource_milestone_events) do |event| - matching_destination_milestone = matching_milestone(event.milestone.title) - - if matching_destination_milestone.present? - event.attributes - .except('id') - .merge(entity_key => new_entity.id, - 'milestone_id' => matching_destination_milestone.id, - 'action' => ResourceMilestoneEvent.actions[event.action], - 'state' => ResourceMilestoneEvent.states[event.state]) + if event.remove? + event_attributes_with_milestone(event, nil) + else + matching_destination_milestone = matching_milestone(event.milestone_title) + + event_attributes_with_milestone(event, matching_destination_milestone) if matching_destination_milestone.present? end end end + def copy_resource_state_events + return unless state_events_supported? + + copy_events(ResourceStateEvent.table_name, original_entity.resource_state_events) do |event| + event.attributes + .except('id') + .merge(entity_key => new_entity.id, + 'state' => ResourceStateEvent.states[event.state]) + end + end + + def event_attributes_with_milestone(event, milestone) + event.attributes + .except('id') + .merge(entity_key => new_entity.id, + 'milestone_id' => milestone&.id, + 'action' => ResourceMilestoneEvent.actions[event.action], + 'state' => ResourceMilestoneEvent.states[event.state]) + end + def copy_events(table_name, events_to_copy) events_to_copy.find_in_batches do |batch| events = batch.map do |event| @@ -94,7 +110,20 @@ module Issuable end def entity_key - new_entity.class.name.parameterize('_').foreign_key + new_entity.class.name.underscore.foreign_key + end + + def milestone_events_supported? + both_respond_to?(:resource_milestone_events) + end + + def state_events_supported? + both_respond_to?(:resource_state_events) + end + + def both_respond_to?(method) + original_entity.respond_to?(method) && + new_entity.respond_to?(method) end end end diff --git a/app/services/issuable/clone/base_service.rb b/app/services/issuable/clone/base_service.rb index 54576e82030..0d1640924e5 100644 --- a/app/services/issuable/clone/base_service.rb +++ b/app/services/issuable/clone/base_service.rb @@ -47,7 +47,7 @@ module Issuable end def new_parent - new_entity.project ? new_entity.project : new_entity.group + new_entity.project || new_entity.group end def group diff --git a/app/services/issuable/common_system_notes_service.rb b/app/services/issuable/common_system_notes_service.rb index 67cf212691f..195616857dc 100644 --- a/app/services/issuable/common_system_notes_service.rb +++ b/app/services/issuable/common_system_notes_service.rb @@ -4,7 +4,7 @@ module Issuable class CommonSystemNotesService < ::BaseService attr_reader :issuable - def execute(issuable, old_labels: [], is_update: true) + def execute(issuable, old_labels: [], old_milestone: nil, is_update: true) @issuable = issuable # We disable touch so that created system notes do not update @@ -22,17 +22,13 @@ module Issuable end create_due_date_note if issuable.previous_changes.include?('due_date') - create_milestone_note if has_milestone_changes? + create_milestone_note(old_milestone) if issuable.previous_changes.include?('milestone_id') create_labels_note(old_labels) if old_labels && issuable.labels != old_labels end end private - def has_milestone_changes? - issuable.previous_changes.include?('milestone_id') - end - def handle_time_tracking_note if issuable.previous_changes.include?('time_estimate') create_time_estimate_note @@ -98,15 +94,19 @@ module Issuable SystemNoteService.change_time_spent(issuable, issuable.project, issuable.time_spent_user) end - def create_milestone_note + def create_milestone_note(old_milestone) if milestone_changes_tracking_enabled? - # Creates a synthetic note - ResourceEvents::ChangeMilestoneService.new(issuable, current_user).execute + create_milestone_change_event(old_milestone) else SystemNoteService.change_milestone(issuable, issuable.project, current_user, issuable.milestone) end end + def create_milestone_change_event(old_milestone) + ResourceEvents::ChangeMilestoneService.new(issuable, current_user, old_milestone: old_milestone) + .execute + end + def milestone_changes_tracking_enabled? ::Feature.enabled?(:track_resource_milestone_change_events, issuable.project) end diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 506f4309a1e..18062bd60da 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -22,7 +22,9 @@ class IssuableBaseService < BaseService params.delete(:milestone_id) params.delete(:labels) params.delete(:add_label_ids) + params.delete(:add_labels) params.delete(:remove_label_ids) + params.delete(:remove_labels) params.delete(:label_ids) params.delete(:assignee_ids) params.delete(:assignee_id) @@ -91,6 +93,8 @@ class IssuableBaseService < BaseService elsif params[label_key] params[label_id_key] = labels_service.find_or_create_by_titles(label_key, find_only: find_only).map(&:id) end + + params.delete(label_key) if params[label_key].nil? end def filter_labels_in_param(key) @@ -217,7 +221,7 @@ class IssuableBaseService < BaseService issuable.assign_attributes(params) if has_title_or_description_changed?(issuable) - issuable.assign_attributes(last_edited_at: Time.now, last_edited_by: current_user) + issuable.assign_attributes(last_edited_at: Time.current, last_edited_by: current_user) end before_update(issuable) @@ -237,7 +241,8 @@ class IssuableBaseService < BaseService end if issuable_saved - Issuable::CommonSystemNotesService.new(project, current_user).execute(issuable, old_labels: old_associations[:labels]) + Issuable::CommonSystemNotesService.new(project, current_user).execute( + issuable, old_labels: old_associations[:labels], old_milestone: old_associations[:milestone]) handle_changes(issuable, old_associations: old_associations) @@ -265,7 +270,7 @@ class IssuableBaseService < BaseService if issuable.changed? || params.present? issuable.assign_attributes(params.merge(updated_by: current_user, - last_edited_at: Time.now, + last_edited_at: Time.current, last_edited_by: current_user)) before_update(issuable, skip_spam_check: true) @@ -360,7 +365,8 @@ class IssuableBaseService < BaseService { labels: issuable.labels.to_a, mentioned_users: issuable.mentioned_users(current_user).to_a, - assignees: issuable.assignees.to_a + assignees: issuable.assignees.to_a, + milestone: issuable.try(:milestone) } associations[:total_time_spent] = issuable.total_time_spent if issuable.respond_to?(:total_time_spent) associations[:description] = issuable.description diff --git a/app/services/issues/build_service.rb b/app/services/issues/build_service.rb index daef468987e..e62315de5f9 100644 --- a/app/services/issues/build_service.rb +++ b/app/services/issues/build_service.rb @@ -65,15 +65,19 @@ module Issues private def whitelisted_issue_params + base_params = [:title, :description, :confidential] + admin_params = [:milestone_id] + if can?(current_user, :admin_issue, project) - params.slice(:title, :description, :milestone_id) + params.slice(*(base_params + admin_params)) else - params.slice(:title, :description) + params.slice(*base_params) end end def build_issue_params - issue_params_with_info_from_discussions.merge(whitelisted_issue_params) + { author: current_user }.merge(issue_params_with_info_from_discussions) + .merge(whitelisted_issue_params) end end end diff --git a/app/services/issues/related_branches_service.rb b/app/services/issues/related_branches_service.rb index 76af482b7ac..46076218857 100644 --- a/app/services/issues/related_branches_service.rb +++ b/app/services/issues/related_branches_service.rb @@ -5,11 +5,29 @@ module Issues class RelatedBranchesService < Issues::BaseService def execute(issue) - branches_with_iid_of(issue) - branches_with_merge_request_for(issue) + branch_names = branches_with_iid_of(issue) - branches_with_merge_request_for(issue) + branch_names.map { |branch_name| branch_data(branch_name) } end private + def branch_data(branch_name) + { + name: branch_name, + pipeline_status: pipeline_status(branch_name) + } + end + + def pipeline_status(branch_name) + branch = project.repository.find_branch(branch_name) + target = branch&.dereferenced_target + + return unless target + + pipeline = project.pipeline_for(branch_name, target.sha) + pipeline.detailed_status(current_user) if can?(current_user, :read_pipeline, pipeline) + end + def branches_with_merge_request_for(issue) Issues::ReferencedMergeRequestsService .new(project, current_user) diff --git a/app/services/issues/update_service.rb b/app/services/issues/update_service.rb index 78ebbd7bff2..ee1a22634af 100644 --- a/app/services/issues/update_service.rb +++ b/app/services/issues/update_service.rb @@ -21,6 +21,10 @@ module Issues spam_check(issue, current_user) unless skip_spam_check end + def after_update(issue) + IssuesChannel.broadcast_to(issue, event: 'updated') if Feature.enabled?(:broadcast_issue_updates, issue.project) + end + def handle_changes(issue, options) old_associations = options.fetch(:old_associations, {}) old_labels = old_associations.fetch(:labels, []) diff --git a/app/services/jira_import/start_import_service.rb b/app/services/jira_import/start_import_service.rb index de4e490281f..59fd463022f 100644 --- a/app/services/jira_import/start_import_service.rb +++ b/app/services/jira_import/start_import_service.rb @@ -56,7 +56,7 @@ module JiraImport import_start_time = Time.zone.now jira_imports_for_project = project.jira_imports.by_jira_project_key(jira_project_key).size + 1 title = "jira-import::#{jira_project_key}-#{jira_imports_for_project}" - description = "Label for issues that were imported from jira on #{import_start_time.strftime('%Y-%m-%d %H:%M:%S')}" + description = "Label for issues that were imported from Jira on #{import_start_time.strftime('%Y-%m-%d %H:%M:%S')}" color = "#{Label.color_for(title)}" { title: title, description: description, color: color } end diff --git a/app/services/lfs/file_transformer.rb b/app/services/lfs/file_transformer.rb index 88f59b820a4..69d33e1c873 100644 --- a/app/services/lfs/file_transformer.rb +++ b/app/services/lfs/file_transformer.rb @@ -5,8 +5,7 @@ module Lfs # return a transformed result with `content` and `encoding` to commit. # # The `repository` passed to the initializer can be a Repository or - # a DesignManagement::Repository (an EE-specific class that inherits - # from Repository). + # class that inherits from Repository. # # The `repository_type` property will be one of the types named in # `Gitlab::GlRepository.types`, and is recorded on the `LfsObjectsProject` diff --git a/app/services/members/request_access_service.rb b/app/services/members/request_access_service.rb index b9b0550e290..4dfedc6cd4e 100644 --- a/app/services/members/request_access_service.rb +++ b/app/services/members/request_access_service.rb @@ -8,7 +8,7 @@ module Members source.members.create( access_level: Gitlab::Access::DEVELOPER, user: current_user, - requested_at: Time.now.utc) + requested_at: Time.current.utc) end private diff --git a/app/services/merge_requests/base_service.rb b/app/services/merge_requests/base_service.rb index 00bf69739ad..7f7bfa29af7 100644 --- a/app/services/merge_requests/base_service.rb +++ b/app/services/merge_requests/base_service.rb @@ -39,6 +39,8 @@ module MergeRequests # Don't try to print expensive instance variables. def inspect + return "#<#{self.class}>" unless respond_to?(:merge_request) + "#<#{self.class} #{merge_request.to_reference(full: true)}>" end @@ -89,8 +91,7 @@ module MergeRequests end def can_use_merge_request_ref?(merge_request) - Feature.enabled?(:ci_use_merge_request_ref, project, default_enabled: true) && - !merge_request.for_fork? + !merge_request.for_fork? end def abort_auto_merge(merge_request, reason) @@ -115,6 +116,32 @@ module MergeRequests yield merge_request end end + + def log_error(exception:, message:, save_message_on_model: false) + reference = merge_request.to_reference(full: true) + data = { + class: self.class.name, + message: message, + merge_request_id: merge_request.id, + merge_request: reference, + save_message_on_model: save_message_on_model + } + + if exception + Gitlab::ErrorTracking.with_context(current_user) do + Gitlab::ErrorTracking.track_exception(exception, data) + end + + data[:"exception.message"] = exception.message + end + + # TODO: Deprecate Gitlab::GitLogger since ErrorTracking should suffice: + # https://gitlab.com/gitlab-org/gitlab/-/issues/216379 + data[:message] = "#{self.class.name} error (#{reference}): #{message}" + Gitlab::GitLogger.error(data) + + merge_request.update(merge_error: message) if save_message_on_model + end end end diff --git a/app/services/merge_requests/rebase_service.rb b/app/services/merge_requests/rebase_service.rb index bc1e97088af..87808a21a15 100644 --- a/app/services/merge_requests/rebase_service.rb +++ b/app/services/merge_requests/rebase_service.rb @@ -2,8 +2,6 @@ module MergeRequests class RebaseService < MergeRequests::BaseService - include Git::Logger - REBASE_ERROR = 'Rebase failed. Please rebase locally' attr_reader :merge_request @@ -22,7 +20,7 @@ module MergeRequests def rebase # Ensure Gitaly isn't already running a rebase if source_project.repository.rebase_in_progress?(merge_request.id) - log_error('Rebase task canceled: Another rebase is already in progress', save_message_on_model: true) + log_error(exception: nil, message: 'Rebase task canceled: Another rebase is already in progress', save_message_on_model: true) return false end @@ -30,8 +28,8 @@ module MergeRequests true rescue => e - log_error(REBASE_ERROR, save_message_on_model: true) - log_error(e.message) + log_error(exception: e, message: REBASE_ERROR, save_message_on_model: true) + false ensure merge_request.update_column(:rebase_jid, nil) diff --git a/app/services/merge_requests/refresh_service.rb b/app/services/merge_requests/refresh_service.rb index c6e1651fa26..56a91fa0305 100644 --- a/app/services/merge_requests/refresh_service.rb +++ b/app/services/merge_requests/refresh_service.rb @@ -115,6 +115,10 @@ module MergeRequests filter_merge_requests(merge_requests).each do |merge_request| if branch_and_project_match?(merge_request) || @push.force_push? merge_request.reload_diff(current_user) + # Clear existing merge error if the push were directed at the + # source branch. Clearing the error when the target branch + # changes will hide the error from the user. + merge_request.merge_error = nil elsif merge_request.merge_request_diff.includes_any_commits?(push_commit_ids) merge_request.reload_diff(current_user) end diff --git a/app/services/merge_requests/squash_service.rb b/app/services/merge_requests/squash_service.rb index d25997c925e..4b04d42b48e 100644 --- a/app/services/merge_requests/squash_service.rb +++ b/app/services/merge_requests/squash_service.rb @@ -2,7 +2,7 @@ module MergeRequests class SquashService < MergeRequests::BaseService - include Git::Logger + SquashInProgressError = Class.new(RuntimeError) def execute # If performing a squash would result in no change, then @@ -11,11 +11,13 @@ module MergeRequests return success(squash_sha: merge_request.diff_head_sha) end - if merge_request.squash_in_progress? + if squash_in_progress? return error(s_('MergeRequests|Squash task canceled: another squash is already in progress.')) end squash! || error(s_('MergeRequests|Failed to squash. Should be done manually.')) + rescue SquashInProgressError + error(s_('MergeRequests|An error occurred while checking whether another squash is in progress.')) end private @@ -25,11 +27,19 @@ module MergeRequests success(squash_sha: squash_sha) rescue => e - log_error("Failed to squash merge request #{merge_request.to_reference(full: true)}:") - log_error(e.message) + log_error(exception: e, message: 'Failed to squash merge request') + false end + def squash_in_progress? + merge_request.squash_in_progress? + rescue => e + log_error(exception: e, message: 'Failed to check squash in progress') + + raise SquashInProgressError, e.message + end + def repository target_project.repository end diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb index c112d75a9b5..514793694ba 100644 --- a/app/services/metrics/dashboard/base_service.rb +++ b/app/services/metrics/dashboard/base_service.rb @@ -42,7 +42,7 @@ module Metrics def allowed? return false unless params[:environment] - Ability.allowed?(current_user, :read_environment, project) + project&.feature_available?(:metrics_dashboard, current_user) end # Returns a new dashboard Hash, supplemented with DB info diff --git a/app/services/metrics/dashboard/grafana_metric_embed_service.rb b/app/services/metrics/dashboard/grafana_metric_embed_service.rb index d58b80162f5..d9ce2c5e905 100644 --- a/app/services/metrics/dashboard/grafana_metric_embed_service.rb +++ b/app/services/metrics/dashboard/grafana_metric_embed_service.rb @@ -18,6 +18,7 @@ module Metrics self.reactive_cache_lease_timeout = 30.seconds self.reactive_cache_refresh_interval = 30.minutes self.reactive_cache_lifetime = 30.minutes + self.reactive_cache_work_type = :external_dependency self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) } class << self @@ -112,7 +113,7 @@ module Metrics end def parse_json(json) - JSON.parse(json, symbolize_names: true) + Gitlab::Json.parse(json, symbolize_names: true) rescue JSON::ParserError raise DashboardProcessingError.new('Grafana response contains invalid json') end diff --git a/app/services/metrics/dashboard/transient_embed_service.rb b/app/services/metrics/dashboard/transient_embed_service.rb index ce81f337e47..cb6ca215447 100644 --- a/app/services/metrics/dashboard/transient_embed_service.rb +++ b/app/services/metrics/dashboard/transient_embed_service.rb @@ -23,7 +23,9 @@ module Metrics override :get_raw_dashboard def get_raw_dashboard - JSON.parse(params[:embed_json]) + Gitlab::Json.parse(params[:embed_json]) + rescue JSON::ParserError => e + invalid_embed_json!(e.message) end override :sequence @@ -35,6 +37,10 @@ module Metrics def identifiers Digest::SHA256.hexdigest(params[:embed_json]) end + + def invalid_embed_json!(message) + raise DashboardProcessingError.new("Parsing error for param :embed_json. #{message}") + end end end end diff --git a/app/services/metrics/users_starred_dashboards/create_service.rb b/app/services/metrics/users_starred_dashboards/create_service.rb new file mode 100644 index 00000000000..7784ed4eb4e --- /dev/null +++ b/app/services/metrics/users_starred_dashboards/create_service.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# Create Metrics::UsersStarredDashboard entry for given user based on matched dashboard_path, project +module Metrics + module UsersStarredDashboards + class CreateService < ::BaseService + include Stepable + + steps :authorize_create_action, + :parse_dashboard_path, + :create + + def initialize(user, project, dashboard_path) + @user, @project, @dashboard_path = user, project, dashboard_path + end + + def execute + keys = %i[status message starred_dashboard] + status, message, dashboards = execute_steps.values_at(*keys) + + if status != :success + ServiceResponse.error(message: message) + else + ServiceResponse.success(payload: dashboards) + end + end + + private + + attr_reader :user, :project, :dashboard_path + + def authorize_create_action(_options) + if Ability.allowed?(user, :create_metrics_user_starred_dashboard, project) + success(user: user, project: project) + else + error(s_('Metrics::UsersStarredDashboards|You are not authorized to add star to this dashboard')) + end + end + + def parse_dashboard_path(options) + if dashboard_path_exists? + options[:dashboard_path] = dashboard_path + success(options) + else + error(s_('Metrics::UsersStarredDashboards|Dashboard with requested path can not be found')) + end + end + + def create(options) + starred_dashboard = build_starred_dashboard_from(options) + + if starred_dashboard.save + success(starred_dashboard: starred_dashboard) + else + error(starred_dashboard.errors.messages) + end + end + + def build_starred_dashboard_from(options) + Metrics::UsersStarredDashboard.new( + user: options.fetch(:user), + project: options.fetch(:project), + dashboard_path: options.fetch(:dashboard_path) + ) + end + + def dashboard_path_exists? + Gitlab::Metrics::Dashboard::Finder + .find_all_paths(project) + .any? { |dashboard| dashboard[:path] == dashboard_path } + end + end + end +end diff --git a/app/services/metrics/users_starred_dashboards/delete_service.rb b/app/services/metrics/users_starred_dashboards/delete_service.rb new file mode 100644 index 00000000000..579715bd49f --- /dev/null +++ b/app/services/metrics/users_starred_dashboards/delete_service.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +# Delete all matching Metrics::UsersStarredDashboard entries for given user based on matched dashboard_path, project +module Metrics + module UsersStarredDashboards + class DeleteService < ::BaseService + def initialize(user, project, dashboard_path = nil) + @user, @project, @dashboard_path = user, project, dashboard_path + end + + def execute + ServiceResponse.success(payload: { deleted_rows: starred_dashboards.delete_all }) + end + + private + + attr_reader :user, :project, :dashboard_path + + def starred_dashboards + # since deleted records are scoped to their owner there is no need to + # check if that user can delete them, also if user lost access to + # project it shouldn't block that user from removing them + dashboards = user.metrics_users_starred_dashboards + + if dashboard_path.present? + dashboards.for_project_dashboard(project, dashboard_path) + else + dashboards.for_project(project) + end + end + end + end +end diff --git a/app/services/namespaces/check_storage_size_service.rb b/app/services/namespaces/check_storage_size_service.rb new file mode 100644 index 00000000000..b3cf17681ee --- /dev/null +++ b/app/services/namespaces/check_storage_size_service.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Namespaces + class CheckStorageSizeService + include ActiveSupport::NumberHelper + include Gitlab::Allowable + include Gitlab::Utils::StrongMemoize + + def initialize(namespace, user) + @root_namespace = namespace.root_ancestor + @root_storage_size = Namespace::RootStorageSize.new(root_namespace) + @user = user + end + + def execute + return ServiceResponse.success unless Feature.enabled?(:namespace_storage_limit, root_namespace) + return ServiceResponse.success if alert_level == :none + + if root_storage_size.above_size_limit? + ServiceResponse.error(message: above_size_limit_message, payload: payload) + else + ServiceResponse.success(payload: payload) + end + end + + private + + attr_reader :root_namespace, :root_storage_size, :user + + USAGE_THRESHOLDS = { + none: 0.0, + info: 0.5, + warning: 0.75, + alert: 0.95, + error: 1.0 + }.freeze + + def payload + return {} unless can?(user, :admin_namespace, root_namespace) + + { + explanation_message: explanation_message, + usage_message: usage_message, + alert_level: alert_level + } + end + + def explanation_message + root_storage_size.above_size_limit? ? above_size_limit_message : below_size_limit_message + end + + def usage_message + s_("You reached %{usage_in_percent} of %{namespace_name}'s capacity (%{used_storage} of %{storage_limit})" % current_usage_params) + end + + def alert_level + strong_memoize(:alert_level) do + usage_ratio = root_storage_size.usage_ratio + current_level = USAGE_THRESHOLDS.each_key.first + + USAGE_THRESHOLDS.each do |level, threshold| + current_level = level if usage_ratio >= threshold + end + + current_level + end + end + + def below_size_limit_message + s_("If you reach 100%% storage capacity, you will not be able to: %{base_message}" % { base_message: base_message } ) + end + + def above_size_limit_message + s_("%{namespace_name} is now read-only. You cannot: %{base_message}" % { namespace_name: root_namespace.name, base_message: base_message }) + end + + def base_message + s_("push to your repository, create pipelines, create issues or add comments. To reduce storage capacity, delete unused repositories, artifacts, wikis, issues, and pipelines.") + end + + def current_usage_params + { + usage_in_percent: number_to_percentage(root_storage_size.usage_ratio * 100, precision: 0), + namespace_name: root_namespace.name, + used_storage: formatted(root_storage_size.current_size), + storage_limit: formatted(root_storage_size.limit) + } + end + + def formatted(number) + number_to_human_size(number, delimiter: ',', precision: 2) + end + end +end diff --git a/app/services/notes/post_process_service.rb b/app/services/notes/post_process_service.rb index 53b3b57f4af..bc86118a150 100644 --- a/app/services/notes/post_process_service.rb +++ b/app/services/notes/post_process_service.rb @@ -16,10 +16,18 @@ module Notes return if @note.for_personal_snippet? @note.create_cross_references! + ::SystemNoteService.design_discussion_added(@note) if create_design_discussion_system_note? + execute_note_hooks end end + private + + def create_design_discussion_system_note? + @note && @note.for_design? && @note.start_of_discussion? + end + def hook_data Gitlab::DataBuilder::Note.build(@note, @note.author) end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 91e19d190bd..4c1db03fab8 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -66,6 +66,14 @@ class NotificationService mailer.access_token_about_to_expire_email(user).deliver_later end + # Notify a user when a previously unknown IP or device is used to + # sign in to their account + def unknown_sign_in(user, ip) + return unless user.can?(:receive_notifications) + + mailer.unknown_sign_in_email(user, ip).deliver_later + end + # When create an issue we should send an email to: # # * issue assignee if their notification level is not Disabled @@ -537,6 +545,18 @@ class NotificationService end end + def group_was_exported(group, current_user) + return true unless notifiable?(current_user, :mention, group: group) + + mailer.group_was_exported_email(current_user, group).deliver_later + end + + def group_was_not_exported(group, current_user, errors) + return true unless notifiable?(current_user, :mention, group: group) + + mailer.group_was_not_exported_email(current_user, group, errors).deliver_later + end + protected def new_resource_email(target, method) diff --git a/app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb b/app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb index 1c03641469e..e14241158a6 100644 --- a/app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb +++ b/app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb @@ -51,8 +51,6 @@ module PagesDomains def save_order_error(acme_order, api_order) log_error(api_order) - return unless Feature.enabled?(:pages_letsencrypt_errors, pages_domain.project) - pages_domain.assign_attributes(auto_ssl_failed: true) pages_domain.save!(validate: false) diff --git a/app/services/pod_logs/base_service.rb b/app/services/pod_logs/base_service.rb index 2451ab8e0ce..8936f9b67a5 100644 --- a/app/services/pod_logs/base_service.rb +++ b/app/services/pod_logs/base_service.rb @@ -58,6 +58,9 @@ module PodLogs result[:pod_name] = params['pod_name'].presence result[:container_name] = params['container_name'].presence + return error(_('Invalid pod_name')) if result[:pod_name] && !result[:pod_name].is_a?(String) + return error(_('Invalid container_name')) if result[:container_name] && !result[:container_name].is_a?(String) + success(result) end diff --git a/app/services/pod_logs/elasticsearch_service.rb b/app/services/pod_logs/elasticsearch_service.rb index aac0fa424ca..f79562c8ab3 100644 --- a/app/services/pod_logs/elasticsearch_service.rb +++ b/app/services/pod_logs/elasticsearch_service.rb @@ -11,6 +11,7 @@ module PodLogs :pod_logs, :filter_return_keys + self.reactive_cache_work_type = :external_dependency self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) } private @@ -52,12 +53,16 @@ module PodLogs def check_search(result) result[:search] = params['search'] if params.key?('search') + return error(_('Invalid search parameter')) if result[:search] && !result[:search].is_a?(String) + success(result) end def check_cursor(result) result[:cursor] = params['cursor'] if params.key?('cursor') + return error(_('Invalid cursor parameter')) if result[:cursor] && !result[:cursor].is_a?(String) + success(result) end @@ -65,6 +70,8 @@ module PodLogs client = cluster&.application_elastic_stack&.elasticsearch_client return error(_('Unable to connect to Elasticsearch')) unless client + chart_above_v2 = cluster.application_elastic_stack.chart_above_v2? + response = ::Gitlab::Elasticsearch::Logs::Lines.new(client).pod_logs( namespace, pod_name: result[:pod_name], @@ -72,7 +79,8 @@ module PodLogs search: result[:search], start_time: result[:start_time], end_time: result[:end_time], - cursor: result[:cursor] + cursor: result[:cursor], + chart_above_v2: chart_above_v2 ) result.merge!(response) diff --git a/app/services/pod_logs/kubernetes_service.rb b/app/services/pod_logs/kubernetes_service.rb index 0a8072a9037..b573ceae1aa 100644 --- a/app/services/pod_logs/kubernetes_service.rb +++ b/app/services/pod_logs/kubernetes_service.rb @@ -17,6 +17,7 @@ module PodLogs :split_logs, :filter_return_keys + self.reactive_cache_work_type = :external_dependency self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) } private @@ -46,6 +47,10 @@ module PodLogs ' chars' % { max_length: K8S_NAME_MAX_LENGTH })) end + unless result[:pod_name] =~ Gitlab::Regex.kubernetes_dns_subdomain_regex + return error(_('pod_name can contain only lowercase letters, digits, \'-\', and \'.\' and must start and end with an alphanumeric character')) + end + unless result[:pods].include?(result[:pod_name]) return error(_('Pod does not exist')) end @@ -69,6 +74,10 @@ module PodLogs ' %{max_length} chars' % { max_length: K8S_NAME_MAX_LENGTH })) end + unless result[:container_name] =~ Gitlab::Regex.kubernetes_dns_subdomain_regex + return error(_('container_name can contain only lowercase letters, digits, \'-\', and \'.\' and must start and end with an alphanumeric character')) + end + unless container_names.include?(result[:container_name]) return error(_('Container does not exist')) end diff --git a/app/services/post_receive_service.rb b/app/services/post_receive_service.rb index f12e45d701a..65e6ebc17d2 100644 --- a/app/services/post_receive_service.rb +++ b/app/services/post_receive_service.rb @@ -29,6 +29,8 @@ class PostReceiveService response.add_alert_message(message) end + response.add_alert_message(storage_size_limit_alert) + broadcast_message = BroadcastMessage.current_banner_messages&.last&.message response.add_alert_message(broadcast_message) @@ -74,4 +76,19 @@ class PostReceiveService ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end + + private + + def storage_size_limit_alert + return unless repository&.repo_type&.project? + + payload = Namespaces::CheckStorageSizeService.new(project.namespace, user).execute.payload + return unless payload.present? + + alert_level = "##### #{payload[:alert_level].to_s.upcase} #####" + + [alert_level, payload[:usage_message], payload[:explanation_message]].join("\n") + end end + +PostReceiveService.prepend_if_ee('EE::PostReceiveService') diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb index 1ce1ef7a1cd..76c89e85f17 100644 --- a/app/services/projects/alerting/notify_service.rb +++ b/app/services/projects/alerting/notify_service.rb @@ -10,7 +10,10 @@ module Projects return forbidden unless alerts_service_activated? return unauthorized unless valid_token?(token) - process_incident_issues if process_issues? + alert = create_alert + return bad_request unless alert.persisted? + + process_incident_issues(alert) if process_issues? send_alert_email if send_email? ServiceResponse.success @@ -22,13 +25,21 @@ module Projects delegate :alerts_service, :alerts_service_activated?, to: :project + def am_alert_params + Gitlab::AlertManagement::AlertParams.from_generic_alert(project: project, payload: params.to_h) + end + + def create_alert + AlertManagement::Alert.create(am_alert_params) + end + def send_email? incident_management_setting.send_email? end - def process_incident_issues + def process_incident_issues(alert) IncidentManagement::ProcessAlertWorker - .perform_async(project.id, parsed_payload) + .perform_async(project.id, parsed_payload, alert.id) end def send_alert_email diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index fc09d14ba4d..b53a9c1561e 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -33,7 +33,7 @@ module Projects end def order_by_date(tags) - now = DateTime.now + now = DateTime.current tags.sort_by { |tag| tag.created_at || now }.reverse end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 429ae905e3d..3233d1799b8 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -108,8 +108,22 @@ module Projects # users in the background def setup_authorizations if @project.group - @project.group.refresh_members_authorized_projects(blocking: false) current_user.refresh_authorized_projects + + if Feature.enabled?(:specialized_project_authorization_workers) + AuthorizedProjectUpdate::ProjectCreateWorker.perform_async(@project.id) + # AuthorizedProjectsWorker uses an exclusive lease per user but + # specialized workers might have synchronization issues. Until we + # compare the inconsistency rates of both approaches, we still run + # AuthorizedProjectsWorker but with some delay and lower urgency as a + # safety net. + @project.group.refresh_members_authorized_projects( + blocking: false, + priority: UserProjectAccessChangedService::LOW_PRIORITY + ) + else + @project.group.refresh_members_authorized_projects(blocking: false) + end else @project.add_maintainer(@project.namespace.owner, current_user: current_user) end @@ -202,8 +216,19 @@ module Projects end end + def extra_attributes_for_measurement + { + current_user: current_user&.name, + project_full_path: "#{project_namespace&.full_path}/#{@params[:path]}" + } + end + private + def project_namespace + @project_namespace ||= Namespace.find_by_id(@params[:namespace_id]) || current_user.namespace + end + def create_from_template? @params[:template_name].present? || @params[:template_project_id].present? end @@ -224,4 +249,9 @@ module Projects end end +# rubocop: disable Cop/InjectEnterpriseEditionModule Projects::CreateService.prepend_if_ee('EE::Projects::CreateService') +# rubocop: enable Cop/InjectEnterpriseEditionModule + +# Measurable should be at the bottom of the ancestor chain, so it will measure execution of EE::Projects::CreateService as well +Projects::CreateService.prepend(Measurable) diff --git a/app/services/projects/gitlab_projects_import_service.rb b/app/services/projects/gitlab_projects_import_service.rb index 234ebbc6651..2e192942b9c 100644 --- a/app/services/projects/gitlab_projects_import_service.rb +++ b/app/services/projects/gitlab_projects_import_service.rb @@ -29,17 +29,21 @@ module Projects end def project_with_same_full_path? - Project.find_by_full_path("#{current_namespace.full_path}/#{params[:path]}").present? + Project.find_by_full_path(project_path).present? end # rubocop: disable CodeReuse/ActiveRecord def current_namespace strong_memoize(:current_namespace) do - Namespace.find_by(id: params[:namespace_id]) + Namespace.find_by(id: params[:namespace_id]) || current_user.namespace end end # rubocop: enable CodeReuse/ActiveRecord + def project_path + "#{current_namespace.full_path}/#{params[:path]}" + end + def overwrite? strong_memoize(:overwrite) do params.delete(:overwrite) diff --git a/app/services/projects/hashed_storage/base_attachment_service.rb b/app/services/projects/hashed_storage/base_attachment_service.rb index f8852c206e3..a2a7895ba17 100644 --- a/app/services/projects/hashed_storage/base_attachment_service.rb +++ b/app/services/projects/hashed_storage/base_attachment_service.rb @@ -70,7 +70,7 @@ module Projects # # @param [String] new_path def discard_path!(new_path) - discarded_path = "#{new_path}-#{Time.now.utc.to_i}" + discarded_path = "#{new_path}-#{Time.current.utc.to_i}" logger.info("Moving existing empty attachments folder from '#{new_path}' to '#{discarded_path}', (PROJECT_ID=#{project.id})") FileUtils.mv(new_path, discarded_path) diff --git a/app/services/projects/hashed_storage/base_repository_service.rb b/app/services/projects/hashed_storage/base_repository_service.rb index d81aa4de9f1..065bf8725be 100644 --- a/app/services/projects/hashed_storage/base_repository_service.rb +++ b/app/services/projects/hashed_storage/base_repository_service.rb @@ -8,13 +8,15 @@ module Projects class BaseRepositoryService < BaseService include Gitlab::ShellAdapter - attr_reader :old_disk_path, :new_disk_path, :old_storage_version, :logger, :move_wiki + attr_reader :old_disk_path, :new_disk_path, :old_storage_version, + :logger, :move_wiki, :move_design def initialize(project:, old_disk_path:, logger: nil) @project = project @logger = logger || Gitlab::AppLogger @old_disk_path = old_disk_path @move_wiki = has_wiki? + @move_design = has_design? end protected @@ -23,6 +25,10 @@ module Projects gitlab_shell.repository_exists?(project.repository_storage, "#{old_wiki_disk_path}.git") end + def has_design? + gitlab_shell.repository_exists?(project.repository_storage, "#{old_design_disk_path}.git") + end + def move_repository(from_name, to_name) from_exists = gitlab_shell.repository_exists?(project.repository_storage, "#{from_name}.git") to_exists = gitlab_shell.repository_exists?(project.repository_storage, "#{to_name}.git") @@ -58,12 +64,18 @@ module Projects project.clear_memoization(:wiki) end + if move_design + result &&= move_repository(old_design_disk_path, new_design_disk_path) + project.clear_memoization(:design_repository) + end + result end def rollback_folder_move move_repository(new_disk_path, old_disk_path) move_repository(new_wiki_disk_path, old_wiki_disk_path) + move_repository(new_design_disk_path, old_design_disk_path) if move_design end def try_to_set_repository_read_only! @@ -87,8 +99,18 @@ module Projects def new_wiki_disk_path @new_wiki_disk_path ||= "#{new_disk_path}#{wiki_path_suffix}" end + + def design_path_suffix + @design_path_suffix ||= ::Gitlab::GlRepository::DESIGN.path_suffix + end + + def old_design_disk_path + @old_design_disk_path ||= "#{old_disk_path}#{design_path_suffix}" + end + + def new_design_disk_path + @new_design_disk_path ||= "#{new_disk_path}#{design_path_suffix}" + end end end end - -Projects::HashedStorage::BaseRepositoryService.prepend_if_ee('EE::Projects::HashedStorage::BaseRepositoryService') diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 8893bf18e1f..86cb4f35206 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -3,19 +3,35 @@ module Projects module ImportExport class ExportService < BaseService - def execute(after_export_strategy = nil, options = {}) + prepend Measurable + + def initialize(*args) + super + + @shared = project.import_export_shared + end + + def execute(after_export_strategy = nil) unless project.template_source? || can?(current_user, :admin_project, project) raise ::Gitlab::ImportExport::Error.permission_error(current_user, project) end - @shared = project.import_export_shared - save_all! execute_after_export_action(after_export_strategy) ensure cleanup end + protected + + def extra_attributes_for_measurement + { + current_user: current_user&.name, + project_full_path: project&.full_path, + file_path: shared.export_path + } + end + private attr_accessor :shared @@ -42,7 +58,10 @@ module Projects end def exporters - [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver, snippets_repo_saver] + [ + version_saver, avatar_saver, project_tree_saver, uploads_saver, + repo_saver, wiki_repo_saver, lfs_saver, snippets_repo_saver, design_repo_saver + ] end def version_saver @@ -81,6 +100,10 @@ module Projects Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: current_user, project: project, shared: shared) end + def design_repo_saver + Gitlab::ImportExport::DesignRepoSaver.new(project: project, shared: shared) + end + def cleanup FileUtils.rm_rf(shared.archive_path) if shared&.archive_path end @@ -103,5 +126,3 @@ module Projects end end end - -Projects::ImportExport::ExportService.prepend_if_ee('EE::Projects::ImportExport::ExportService') diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index 4b294a97516..449c4c3de6b 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -3,6 +3,7 @@ module Projects class ImportService < BaseService Error = Class.new(StandardError) + PermissionError = Class.new(StandardError) # Returns true if this importer is supposed to perform its work in the # background. @@ -21,6 +22,8 @@ module Projects import_data + after_execute_hook + success rescue Gitlab::UrlBlocker::BlockedUrlError => e Gitlab::ErrorTracking.track_exception(e, project_path: project.full_path, importer: project.import_type) @@ -34,8 +37,23 @@ module Projects error(s_("ImportProjects|Error importing repository %{project_safe_import_url} into %{project_full_path} - %{message}") % { project_safe_import_url: project.safe_import_url, project_full_path: project.full_path, message: message }) end + protected + + def extra_attributes_for_measurement + { + current_user: current_user&.name, + project_full_path: project&.full_path, + import_type: project&.import_type, + file_path: project&.import_source + } + end + private + def after_execute_hook + # Defined in EE::Projects::ImportService + end + def add_repository_to_project if project.external_import? && !unknown_url? begin @@ -130,3 +148,10 @@ module Projects end end end + +# rubocop: disable Cop/InjectEnterpriseEditionModule +Projects::ImportService.prepend_if_ee('EE::Projects::ImportService') +# rubocop: enable Cop/InjectEnterpriseEditionModule + +# Measurable should be at the bottom of the ancestor chain, so it will measure execution of EE::Projects::ImportService as well +Projects::ImportService.prepend(Measurable) diff --git a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb index 48a21bf94ba..efd410088ab 100644 --- a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb +++ b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb @@ -69,7 +69,7 @@ module Projects # application/vnd.git-lfs+json # (https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md#requests), # HTTParty does not know this is actually JSON. - data = JSON.parse(response.body) + data = Gitlab::Json.parse(response.body) raise DownloadLinksError, "LFS Batch API did return any objects" unless data.is_a?(Hash) && data.key?('objects') diff --git a/app/services/projects/lsif_data_service.rb b/app/services/projects/lsif_data_service.rb index 142a5a910d4..5e7055b3309 100644 --- a/app/services/projects/lsif_data_service.rb +++ b/app/services/projects/lsif_data_service.rb @@ -42,7 +42,7 @@ module Projects file.open do |stream| Zlib::GzipReader.wrap(stream) do |gz_stream| - data = JSON.parse(gz_stream.read) + data = Gitlab::Json.parse(gz_stream.read) end end diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index 6ebc061c2e3..2583a6cae9f 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -12,6 +12,7 @@ module Projects return unprocessable_entity unless valid_version? return unauthorized unless valid_alert_manager_token?(token) + process_prometheus_alerts persist_events send_alert_email if send_email? process_incident_issues if process_issues? @@ -115,6 +116,14 @@ module Projects end end + def process_prometheus_alerts + alerts.each do |alert| + AlertManagement::ProcessPrometheusAlertService + .new(project, nil, alert.to_h) + .execute + end + end + def persist_events CreateEventsService.new(project, nil, params).execute end diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb index 6013b00b8c6..0483c951f1e 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_service_template.rb @@ -4,8 +4,10 @@ module Projects class PropagateServiceTemplate BATCH_SIZE = 100 - def self.propagate(*args) - new(*args).propagate + delegate :data_fields_present?, to: :template + + def self.propagate(template) + new(template).propagate end def initialize(template) @@ -13,15 +15,15 @@ module Projects end def propagate - return unless @template.active? - - Rails.logger.info("Propagating services for template #{@template.id}") # rubocop:disable Gitlab/RailsLogger + return unless template.active? propagate_projects_with_template end private + attr_reader :template + def propagate_projects_with_template loop do batch = Project.uncached { project_ids_batch } @@ -38,7 +40,14 @@ module Projects end Project.transaction do - bulk_insert_services(service_hash.keys << 'project_id', service_list) + results = bulk_insert(Service, service_hash.keys << 'project_id', service_list) + + if data_fields_present? + data_list = results.map { |row| data_hash.values << row['id'] } + + bulk_insert(template.data_fields.class, data_hash.keys << 'service_id', data_list) + end + run_callbacks(batch) end end @@ -52,36 +61,27 @@ module Projects SELECT true FROM services WHERE services.project_id = projects.id - AND services.type = '#{@template.type}' + AND services.type = #{ActiveRecord::Base.connection.quote(template.type)} ) AND projects.pending_delete = false AND projects.archived = false LIMIT #{BATCH_SIZE} - SQL + SQL ) end - def bulk_insert_services(columns, values_array) - ActiveRecord::Base.connection.execute( - <<-SQL.strip_heredoc - INSERT INTO services (#{columns.join(', ')}) - VALUES #{values_array.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} - SQL - ) + def bulk_insert(klass, columns, values_array) + items_to_insert = values_array.map { |array| Hash[columns.zip(array)] } + + klass.insert_all(items_to_insert, returning: [:id]) end def service_hash - @service_hash ||= - begin - template_hash = @template.as_json(methods: :type).except('id', 'template', 'project_id') - - template_hash.each_with_object({}) do |(key, value), service_hash| - value = value.is_a?(Hash) ? value.to_json : value + @service_hash ||= template.as_json(methods: :type, except: %w[id template project_id]) + end - service_hash[ActiveRecord::Base.connection.quote_column_name(key)] = - ActiveRecord::Base.connection.quote(value) - end - end + def data_hash + @data_hash ||= template.data_fields.as_json(only: template.data_fields.class.column_names).except('id', 'service_id') end # rubocop: disable CodeReuse/ActiveRecord @@ -97,11 +97,11 @@ module Projects # rubocop: enable CodeReuse/ActiveRecord def active_external_issue_tracker? - @template.issue_tracker? && !@template.default + template.issue_tracker? && !template.default end def active_external_wiki? - @template.type == 'ExternalWikiService' + template.type == 'ExternalWikiService' end end end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 309eab59463..60e5b7e2639 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -135,7 +135,8 @@ module Projects return if project.hashed_storage?(:repository) move_repo_folder(@new_path, @old_path) - move_repo_folder("#{@new_path}.wiki", "#{@old_path}.wiki") + move_repo_folder(new_wiki_repo_path, old_wiki_repo_path) + move_repo_folder(new_design_repo_path, old_design_repo_path) end def move_repo_folder(from_name, to_name) @@ -157,8 +158,9 @@ module Projects # Disk path is changed; we need to ensure we reload it project.reload_repository! - # Move wiki repo also if present - move_repo_folder("#{@old_path}.wiki", "#{@new_path}.wiki") + # Move wiki and design repos also if present + move_repo_folder(old_wiki_repo_path, new_wiki_repo_path) + move_repo_folder(old_design_repo_path, new_design_repo_path) end def move_project_uploads(project) @@ -170,6 +172,22 @@ module Projects @new_namespace.full_path ) end + + def old_wiki_repo_path + "#{old_path}#{::Gitlab::GlRepository::WIKI.path_suffix}" + end + + def new_wiki_repo_path + "#{new_path}#{::Gitlab::GlRepository::WIKI.path_suffix}" + end + + def old_design_repo_path + "#{old_path}#{::Gitlab::GlRepository::DESIGN.path_suffix}" + end + + def new_design_repo_path + "#{new_path}#{::Gitlab::GlRepository::DESIGN.path_suffix}" + end end end diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 13a467a3ef9..e554bed6819 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -29,14 +29,16 @@ module Projects remote_mirror.ensure_remote! repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true) - opts = {} - if remote_mirror.only_protected_branches? - opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) - end + response = remote_mirror.update_repository - remote_mirror.update_repository(opts) + if response.divergent_refs.any? + message = "Some refs have diverged and have not been updated on the remote:" + message += "\n\n#{response.divergent_refs.join("\n")}" - remote_mirror.update_finish! + remote_mirror.mark_as_failed!(message) + else + remote_mirror.update_finish! + end end def retry_or_fail(mirror, message, tries) diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb index 2e5de9411d1..0632df6f6d7 100644 --- a/app/services/projects/update_repository_storage_service.rb +++ b/app/services/projects/update_repository_storage_service.rb @@ -1,37 +1,49 @@ # frozen_string_literal: true module Projects - class UpdateRepositoryStorageService < BaseService - include Gitlab::ShellAdapter - + class UpdateRepositoryStorageService Error = Class.new(StandardError) SameFilesystemError = Class.new(Error) - def initialize(project) - @project = project + attr_reader :repository_storage_move + delegate :project, :destination_storage_name, to: :repository_storage_move + delegate :repository, to: :project + + def initialize(repository_storage_move) + @repository_storage_move = repository_storage_move end - def execute(new_repository_storage_key) - raise SameFilesystemError if same_filesystem?(project.repository.storage, new_repository_storage_key) + def execute + repository_storage_move.start! - mirror_repositories(new_repository_storage_key) + raise SameFilesystemError if same_filesystem?(repository.storage, destination_storage_name) - mark_old_paths_for_archive + mirror_repositories - project.update(repository_storage: new_repository_storage_key, repository_read_only: false) - project.leave_pool_repository - project.track_project_repository + project.transaction do + mark_old_paths_for_archive + + repository_storage_move.finish! + project.update!(repository_storage: destination_storage_name, repository_read_only: false) + project.leave_pool_repository + project.track_project_repository + end enqueue_housekeeping - success + ServiceResponse.success - rescue Error, ArgumentError, Gitlab::Git::BaseError => e - project.update(repository_read_only: false) + rescue StandardError => e + project.transaction do + repository_storage_move.do_fail! + project.update!(repository_read_only: false) + end Gitlab::ErrorTracking.track_exception(e, project_path: project.full_path) - error(s_("UpdateRepositoryStorage|Error moving repository storage for %{project_full_path} - %{message}") % { project_full_path: project.full_path, message: e.message }) + ServiceResponse.error( + message: s_("UpdateRepositoryStorage|Error moving repository storage for %{project_full_path} - %{message}") % { project_full_path: project.full_path, message: e.message } + ) end private @@ -40,15 +52,19 @@ module Projects Gitlab::GitalyClient.filesystem_id(old_storage) == Gitlab::GitalyClient.filesystem_id(new_storage) end - def mirror_repositories(new_repository_storage_key) - mirror_repository(new_repository_storage_key) + def mirror_repositories + mirror_repository if project.wiki.repository_exists? - mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI) + mirror_repository(type: Gitlab::GlRepository::WIKI) + end + + if project.design_repository.exists? + mirror_repository(type: ::Gitlab::GlRepository::DESIGN) end end - def mirror_repository(new_storage_key, type: Gitlab::GlRepository::PROJECT) + def mirror_repository(type: Gitlab::GlRepository::PROJECT) unless wait_for_pushes(type) raise Error, s_('UpdateRepositoryStorage|Timeout waiting for %{type} repository pushes') % { type: type.name } end @@ -60,7 +76,7 @@ module Projects # Initialize a git repository on the target path new_repository = Gitlab::Git::Repository.new( - new_storage_key, + destination_storage_name, raw_repository.relative_path, raw_repository.gl_repository, full_path @@ -94,11 +110,18 @@ module Projects wiki.disk_path, "#{new_project_path}.wiki") end + + if design_repository.exists? + GitlabShellWorker.perform_async(:mv_repository, + old_repository_storage, + design_repository.disk_path, + "#{new_project_path}.design") + end end end def moved_path(path) - "#{path}+#{project.id}+moved+#{Time.now.to_i}" + "#{path}+#{project.id}+moved+#{Time.current.to_i}" end # The underlying FetchInternalRemote call uses a `git fetch` to move data @@ -128,5 +151,3 @@ module Projects end end end - -Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService') diff --git a/app/services/prometheus/proxy_service.rb b/app/services/prometheus/proxy_service.rb index 99c739a630b..085cfc76196 100644 --- a/app/services/prometheus/proxy_service.rb +++ b/app/services/prometheus/proxy_service.rb @@ -17,6 +17,7 @@ module Prometheus # is expected to change *and* be fetched again by the frontend self.reactive_cache_refresh_interval = 90.seconds self.reactive_cache_lifetime = 1.minute + self.reactive_cache_work_type = :external_dependency self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) } attr_accessor :proxyable, :method, :path, :params diff --git a/app/services/prometheus/proxy_variable_substitution_service.rb b/app/services/prometheus/proxy_variable_substitution_service.rb index 240586c8419..aa3a09ba05c 100644 --- a/app/services/prometheus/proxy_variable_substitution_service.rb +++ b/app/services/prometheus/proxy_variable_substitution_service.rb @@ -4,11 +4,20 @@ module Prometheus class ProxyVariableSubstitutionService < BaseService include Stepable + VARIABLE_INTERPOLATION_REGEX = / + {{ # Variable needs to be wrapped in these chars. + \s* # Allow whitespace before and after the variable name. + (?<variable> # Named capture. + \w+ # Match one or more word characters. + ) + \s* + }} + /x.freeze + steps :validate_variables, :add_params_to_result, :substitute_params, - :substitute_ruby_variables, - :substitute_liquid_variables + :substitute_variables def initialize(environment, params = {}) @environment, @params = environment, params.deep_dup @@ -46,37 +55,28 @@ module Prometheus success(result) end - def substitute_liquid_variables(result) + def substitute_variables(result) return success(result) unless query(result) - result[:params][:query] = - TemplateEngines::LiquidService.new(query(result)).render(full_context) + result[:params][:query] = gsub(query(result), full_context) success(result) - rescue TemplateEngines::LiquidService::RenderError => e - error(e.message) end - def substitute_ruby_variables(result) - return success(result) unless query(result) - - # The % operator doesn't replace variables if the hash contains string - # keys. - result[:params][:query] = query(result) % predefined_context.symbolize_keys - - success(result) - rescue TypeError, ArgumentError => exception - log_error(exception.message) - Gitlab::ErrorTracking.track_exception(exception, { - template_string: query(result), - variables: predefined_context - }) - - error(_('Malformed string')) + def gsub(string, context) + # Search for variables of the form `{{variable}}` in the string and replace + # them with their value. + string.gsub(VARIABLE_INTERPOLATION_REGEX) do |match| + # Replace with the value of the variable, or if there is no such variable, + # replace the invalid variable with itself. So, + # `up{instance="{{invalid_variable}}"}` will remain + # `up{instance="{{invalid_variable}}"}` after substitution. + context.fetch($~[:variable], match) + end end def predefined_context - @predefined_context ||= Gitlab::Prometheus::QueryVariables.call(@environment) + Gitlab::Prometheus::QueryVariables.call(@environment).stringify_keys end def full_context diff --git a/app/services/releases/create_service.rb b/app/services/releases/create_service.rb index 9a0a876454f..81ca9d6d123 100644 --- a/app/services/releases/create_service.rb +++ b/app/services/releases/create_service.rb @@ -47,11 +47,17 @@ module Releases release.save! + notify_create_release(release) + success(tag: tag, release: release) rescue => e error(e.message, 400) end + def notify_create_release(release) + NotificationService.new.async.send_new_release_notifications(release) + end + def build_release(tag) project.releases.build( name: name, diff --git a/app/services/resources/create_access_token_service.rb b/app/services/resource_access_tokens/create_service.rb index fd3c8d78e58..c8e86e68383 100644 --- a/app/services/resources/create_access_token_service.rb +++ b/app/services/resource_access_tokens/create_service.rb @@ -1,13 +1,11 @@ # frozen_string_literal: true -module Resources - class CreateAccessTokenService < BaseService - attr_accessor :resource_type, :resource - - def initialize(resource_type, resource, user, params = {}) - @resource_type = resource_type +module ResourceAccessTokens + class CreateService < BaseService + def initialize(current_user, resource, params = {}) + @resource_type = resource.class.name.downcase @resource = resource - @current_user = user + @current_user = current_user @params = params.dup end @@ -33,6 +31,8 @@ module Resources private + attr_reader :resource_type, :resource + def feature_enabled? ::Feature.enabled?(:resource_access_token, resource) end @@ -85,7 +85,7 @@ module Resources def personal_access_token_params { - name: "#{resource_type}_bot", + name: params[:name] || "#{resource_type}_bot", impersonation: false, scopes: params[:scopes] || default_scopes, expires_at: params[:expires_at] || nil @@ -93,7 +93,7 @@ module Resources end def default_scopes - Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth.registry_scopes - [:read_user] + Gitlab::Auth.resource_bot_scopes end def provision_access(resource, user) diff --git a/app/services/resource_access_tokens/revoke_service.rb b/app/services/resource_access_tokens/revoke_service.rb new file mode 100644 index 00000000000..eea6bff572b --- /dev/null +++ b/app/services/resource_access_tokens/revoke_service.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module ResourceAccessTokens + class RevokeService < BaseService + include Gitlab::Utils::StrongMemoize + + RevokeAccessTokenError = Class.new(RuntimeError) + + def initialize(current_user, resource, access_token) + @current_user = current_user + @access_token = access_token + @bot_user = access_token.user + @resource = resource + end + + def execute + return error("Failed to find bot user") unless find_member + + PersonalAccessToken.transaction do + access_token.revoke! + + raise RevokeAccessTokenError, "Failed to remove #{bot_user.name} member from: #{resource.name}" unless remove_member + + raise RevokeAccessTokenError, "Migration to ghost user failed" unless migrate_to_ghost_user + end + + success("Revoked access token: #{access_token.name}") + rescue ActiveRecord::ActiveRecordError, RevokeAccessTokenError => error + log_error("Failed to revoke access token for #{bot_user.name}: #{error.message}") + error(error.message) + end + + private + + attr_reader :current_user, :access_token, :bot_user, :resource + + def remove_member + ::Members::DestroyService.new(current_user).execute(find_member) + end + + def migrate_to_ghost_user + ::Users::MigrateToGhostUserService.new(bot_user).execute + end + + def find_member + strong_memoize(:member) do + if resource.is_a?(Project) + resource.project_member(bot_user) + elsif resource.is_a?(Group) + resource.group_member(bot_user) + else + false + end + end + end + + def error(message) + ServiceResponse.error(message: message) + end + + def success(message) + ServiceResponse.success(message: message) + end + end +end diff --git a/app/services/resource_events/base_synthetic_notes_builder_service.rb b/app/services/resource_events/base_synthetic_notes_builder_service.rb index 1b85ca811a1..db8bf6e4b74 100644 --- a/app/services/resource_events/base_synthetic_notes_builder_service.rb +++ b/app/services/resource_events/base_synthetic_notes_builder_service.rb @@ -26,7 +26,7 @@ module ResourceEvents def since_fetch_at(events) return events unless params[:last_fetched_at].present? - last_fetched_at = Time.at(params.fetch(:last_fetched_at).to_i) + last_fetched_at = Time.zone.at(params.fetch(:last_fetched_at).to_i) events.created_after(last_fetched_at - NotesFinder::FETCH_OVERLAP) end diff --git a/app/services/resource_events/change_milestone_service.rb b/app/services/resource_events/change_milestone_service.rb index ea196822f74..82c3e2acad5 100644 --- a/app/services/resource_events/change_milestone_service.rb +++ b/app/services/resource_events/change_milestone_service.rb @@ -2,13 +2,14 @@ module ResourceEvents class ChangeMilestoneService - attr_reader :resource, :user, :event_created_at, :milestone + attr_reader :resource, :user, :event_created_at, :milestone, :old_milestone - def initialize(resource, user, created_at: Time.now) + def initialize(resource, user, created_at: Time.current, old_milestone:) @resource = resource @user = user @event_created_at = created_at @milestone = resource&.milestone + @old_milestone = old_milestone end def execute @@ -26,7 +27,7 @@ module ResourceEvents { user_id: user.id, created_at: event_created_at, - milestone_id: milestone&.id, + milestone_id: action == :add ? milestone&.id : old_milestone&.id, state: ResourceMilestoneEvent.states[resource.state], action: ResourceMilestoneEvent.actions[action], key => resource.id diff --git a/app/services/search/snippet_service.rb b/app/services/search/snippet_service.rb index e686d3bf7c2..30401b28571 100644 --- a/app/services/search/snippet_service.rb +++ b/app/services/search/snippet_service.rb @@ -7,7 +7,7 @@ module Search end def scope - @scope ||= %w[snippet_titles].delete(params[:scope]) { 'snippet_blobs' } + @scope ||= 'snippet_titles' end end end diff --git a/app/services/search_service.rb b/app/services/search_service.rb index c96599f9958..bf21eba28f7 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -6,6 +6,9 @@ class SearchService SEARCH_TERM_LIMIT = 64 SEARCH_CHAR_LIMIT = 4096 + DEFAULT_PER_PAGE = Gitlab::SearchResults::DEFAULT_PER_PAGE + MAX_PER_PAGE = 200 + def initialize(current_user, params = {}) @current_user = current_user @params = params.dup @@ -60,11 +63,19 @@ class SearchService end def search_objects - @search_objects ||= redact_unauthorized_results(search_results.objects(scope, params[:page])) + @search_objects ||= redact_unauthorized_results(search_results.objects(scope, page: params[:page], per_page: per_page)) end private + def per_page + per_page_param = params[:per_page].to_i + + return DEFAULT_PER_PAGE unless per_page_param.positive? + + [MAX_PER_PAGE, per_page_param].min + end + def visible_result?(object) return true unless object.respond_to?(:to_ability_name) && DeclarativePolicy.has_policy?(object) @@ -75,13 +86,13 @@ class SearchService results = results_collection.to_a permitted_results = results.select { |object| visible_result?(object) } - filtered_results = (results - permitted_results).each_with_object({}) do |object, memo| + redacted_results = (results - permitted_results).each_with_object({}) do |object, memo| memo[object.id] = { ability: :"read_#{object.to_ability_name}", id: object.id, class_name: object.class.name } end - log_redacted_search_results(filtered_results.values) if filtered_results.any? + log_redacted_search_results(redacted_results.values) if redacted_results.any? - return results_collection.id_not_in(filtered_results.keys) if results_collection.is_a?(ActiveRecord::Relation) + return results_collection.id_not_in(redacted_results.keys) if results_collection.is_a?(ActiveRecord::Relation) Kaminari.paginate_array( permitted_results, diff --git a/app/services/snippets/base_service.rb b/app/services/snippets/base_service.rb index 2b450db0b83..81d12997335 100644 --- a/app/services/snippets/base_service.rb +++ b/app/services/snippets/base_service.rb @@ -2,8 +2,32 @@ module Snippets class BaseService < ::BaseService + include SpamCheckMethods + + CreateRepositoryError = Class.new(StandardError) + + attr_reader :uploaded_files + + def initialize(project, user = nil, params = {}) + super + + @uploaded_files = Array(@params.delete(:files).presence) + + filter_spam_check_params + end + private + def visibility_allowed?(snippet, visibility_level) + Gitlab::VisibilityLevel.allowed_for?(current_user, visibility_level) + end + + def error_forbidden_visibility(snippet) + deny_visibility_level(snippet) + + snippet_error_response(snippet, 403) + end + def snippet_error_response(snippet, http_status) ServiceResponse.error( message: snippet.errors.full_messages.to_sentence, @@ -11,5 +35,22 @@ module Snippets payload: { snippet: snippet } ) end + + def add_snippet_repository_error(snippet:, error:) + message = repository_error_message(error) + + snippet.errors.add(:repository, message) + end + + def repository_error_message(error) + message = self.is_a?(Snippets::CreateService) ? _("Error creating the snippet") : _("Error updating the snippet") + + # We only want to include additional error detail in the message + # if the error is not a CommitError because we cannot guarantee the message + # will be user-friendly + message += " - #{error.message}" unless error.instance_of?(SnippetRepository::CommitError) + + message + end end end diff --git a/app/services/snippets/create_service.rb b/app/services/snippets/create_service.rb index 155013db344..ed6da3a0ad0 100644 --- a/app/services/snippets/create_service.rb +++ b/app/services/snippets/create_service.rb @@ -2,23 +2,11 @@ module Snippets class CreateService < Snippets::BaseService - include SpamCheckMethods - - CreateRepositoryError = Class.new(StandardError) - def execute - filter_spam_check_params - - @snippet = if project - project.snippets.build(params) - else - PersonalSnippet.new(params) - end - - unless Gitlab::VisibilityLevel.allowed_for?(current_user, @snippet.visibility_level) - deny_visibility_level(@snippet) + @snippet = build_from_params - return snippet_error_response(@snippet, 403) + unless visibility_allowed?(@snippet, @snippet.visibility_level) + return error_forbidden_visibility(@snippet) end @snippet.author = current_user @@ -29,6 +17,8 @@ module Snippets UserAgentDetailService.new(@snippet, @request).create Gitlab::UsageDataCounters::SnippetCounter.count(:create) + move_temporary_files + ServiceResponse.success(payload: { snippet: @snippet } ) else snippet_error_response(@snippet, 400) @@ -37,10 +27,18 @@ module Snippets private + def build_from_params + if project + project.snippets.build(params) + else + PersonalSnippet.new(params) + end + end + def save_and_commit snippet_saved = @snippet.save - if snippet_saved && Feature.enabled?(:version_snippets, current_user) + if snippet_saved create_repository create_commit end @@ -60,7 +58,7 @@ module Snippets @snippet = @snippet.dup end - @snippet.errors.add(:base, e.message) + add_snippet_repository_error(snippet: @snippet, error: e) false end @@ -83,5 +81,13 @@ module Snippets def snippet_files [{ file_path: params[:file_name], content: params[:content] }] end + + def move_temporary_files + return unless @snippet.is_a?(PersonalSnippet) + + uploaded_files.each do |file| + FileMover.new(file, from_model: current_user, to_model: @snippet).execute + end + end end end diff --git a/app/services/snippets/update_service.rb b/app/services/snippets/update_service.rb index e56b20c6057..2dc9266dbd0 100644 --- a/app/services/snippets/update_service.rb +++ b/app/services/snippets/update_service.rb @@ -2,24 +2,15 @@ module Snippets class UpdateService < Snippets::BaseService - include SpamCheckMethods + COMMITTABLE_ATTRIBUTES = %w(file_name content).freeze UpdateError = Class.new(StandardError) - CreateRepositoryError = Class.new(StandardError) def execute(snippet) - # check that user is allowed to set specified visibility_level - new_visibility = visibility_level - - if new_visibility && new_visibility.to_i != snippet.visibility_level - unless Gitlab::VisibilityLevel.allowed_for?(current_user, new_visibility) - deny_visibility_level(snippet, new_visibility) - - return snippet_error_response(snippet, 403) - end + if visibility_changed?(snippet) && !visibility_allowed?(snippet, visibility_level) + return error_forbidden_visibility(snippet) end - filter_spam_check_params snippet.assign_attributes(params) spam_check(snippet, current_user) @@ -34,30 +25,32 @@ module Snippets private + def visibility_changed?(snippet) + visibility_level && visibility_level.to_i != snippet.visibility_level + end + def save_and_commit(snippet) return false unless snippet.save - # In order to avoid non migrated snippets scenarios, - # if the snippet does not have a repository we created it - # We don't need to check if the repository exists - # because `create_repository` already handles it - if Feature.enabled?(:version_snippets, current_user) - create_repository_for(snippet) - end + # If the updated attributes does not need to update + # the repository we can just return + return true unless committable_attributes? - # If the snippet repository exists we commit always - # the changes - create_commit(snippet) if snippet.repository_exists? + create_repository_for(snippet) + create_commit(snippet) true rescue => e - # Restore old attributes + # Restore old attributes but re-assign changes so they're not lost unless snippet.previous_changes.empty? snippet.previous_changes.each { |attr, value| snippet[attr] = value[0] } snippet.save + + snippet.assign_attributes(params) end - snippet.errors.add(:repository, 'Error updating the snippet') + add_snippet_repository_error(snippet: snippet, error: e) + log_error(e.message) # If the commit action failed we remove it because @@ -92,7 +85,7 @@ module Snippets end def snippet_files(snippet) - [{ previous_path: snippet.blobs.first&.path, + [{ previous_path: snippet.file_name_on_repo, file_path: params[:file_name], content: params[:content] }] end @@ -104,5 +97,9 @@ module Snippets def repository_empty?(snippet) snippet.repository._uncached_exists? && !snippet.repository._uncached_has_visible_content? end + + def committable_attributes? + (params.stringify_keys.keys & COMMITTABLE_ATTRIBUTES).present? + end end end diff --git a/app/services/spam/akismet_service.rb b/app/services/spam/akismet_service.rb index 7d16743b3ed..ab35fb8700f 100644 --- a/app/services/spam/akismet_service.rb +++ b/app/services/spam/akismet_service.rb @@ -17,7 +17,7 @@ module Spam params = { type: 'comment', text: text, - created_at: DateTime.now, + created_at: DateTime.current, author: owner_name, author_email: owner_email, referrer: options[:referrer] diff --git a/app/services/spam/spam_action_service.rb b/app/services/spam/spam_action_service.rb new file mode 100644 index 00000000000..f0a4aff4443 --- /dev/null +++ b/app/services/spam/spam_action_service.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Spam + class SpamActionService + include SpamConstants + + attr_accessor :target, :request, :options + attr_reader :spam_log + + def initialize(spammable:, request:) + @target = spammable + @request = request + @options = {} + + if @request + @options[:ip_address] = @request.env['action_dispatch.remote_ip'].to_s + @options[:user_agent] = @request.env['HTTP_USER_AGENT'] + @options[:referrer] = @request.env['HTTP_REFERRER'] + else + @options[:ip_address] = @target.ip_address + @options[:user_agent] = @target.user_agent + end + end + + def execute(api: false, recaptcha_verified:, spam_log_id:, user:) + if recaptcha_verified + # If it's a request which is already verified through reCAPTCHA, + # update the spam log accordingly. + SpamLog.verify_recaptcha!(user_id: user.id, id: spam_log_id) + else + return if allowlisted?(user) + return unless request + return unless check_for_spam? + + perform_spam_service_check(api) + end + end + + delegate :check_for_spam?, to: :target + + private + + def allowlisted?(user) + user.respond_to?(:gitlab_employee) && user.gitlab_employee? + end + + def perform_spam_service_check(api) + # since we can check for spam, and recaptcha is not verified, + # ask the SpamVerdictService what to do with the target. + spam_verdict_service.execute.tap do |result| + case result + when REQUIRE_RECAPTCHA + create_spam_log(api) + + break if target.allow_possible_spam? + + target.needs_recaptcha! + when DISALLOW + # TODO: remove `unless target.allow_possible_spam?` once this flag has been passed to `SpamVerdictService` + # https://gitlab.com/gitlab-org/gitlab/-/issues/214739 + target.spam! unless target.allow_possible_spam? + create_spam_log(api) + when ALLOW + target.clear_spam_flags! + end + end + end + + def create_spam_log(api) + @spam_log = SpamLog.create!( + { + user_id: target.author_id, + title: target.spam_title, + description: target.spam_description, + source_ip: options[:ip_address], + user_agent: options[:user_agent], + noteable_type: target.class.to_s, + via_api: api + } + ) + + target.spam_log = spam_log + end + + def spam_verdict_service + SpamVerdictService.new(target: target, + request: @request, + options: options) + end + end +end diff --git a/app/services/spam/spam_check_service.rb b/app/services/spam/spam_check_service.rb deleted file mode 100644 index 3269f9d687a..00000000000 --- a/app/services/spam/spam_check_service.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -module Spam - class SpamCheckService - include AkismetMethods - - attr_accessor :target, :request, :options - attr_reader :spam_log - - def initialize(spammable:, request:) - @target = spammable - @request = request - @options = {} - - if @request - @options[:ip_address] = @request.env['action_dispatch.remote_ip'].to_s - @options[:user_agent] = @request.env['HTTP_USER_AGENT'] - @options[:referrer] = @request.env['HTTP_REFERRER'] - else - @options[:ip_address] = @target.ip_address - @options[:user_agent] = @target.user_agent - end - end - - def execute(api: false, recaptcha_verified:, spam_log_id:, user_id:) - if recaptcha_verified - # If it's a request which is already verified through recaptcha, - # update the spam log accordingly. - SpamLog.verify_recaptcha!(user_id: user_id, id: spam_log_id) - else - # Otherwise, it goes to Akismet for spam check. - # If so, it assigns spammable object as "spam" and creates a SpamLog record. - possible_spam = check(api) - target.spam = possible_spam unless target.allow_possible_spam? - target.spam_log = spam_log - end - end - - private - - def check(api) - return unless request - return unless check_for_spam? - return unless akismet.spam? - - create_spam_log(api) - true - end - - def check_for_spam? - target.check_for_spam? - end - - def create_spam_log(api) - @spam_log = SpamLog.create!( - { - user_id: target.author_id, - title: target.spam_title, - description: target.spam_description, - source_ip: options[:ip_address], - user_agent: options[:user_agent], - noteable_type: target.class.to_s, - via_api: api - } - ) - end - end -end diff --git a/app/services/spam/spam_constants.rb b/app/services/spam/spam_constants.rb new file mode 100644 index 00000000000..085bac684c4 --- /dev/null +++ b/app/services/spam/spam_constants.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Spam + module SpamConstants + REQUIRE_RECAPTCHA = :recaptcha + DISALLOW = :disallow + ALLOW = :allow + end +end diff --git a/app/services/spam/spam_verdict_service.rb b/app/services/spam/spam_verdict_service.rb new file mode 100644 index 00000000000..2b4d5f4a984 --- /dev/null +++ b/app/services/spam/spam_verdict_service.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Spam + class SpamVerdictService + include AkismetMethods + include SpamConstants + + def initialize(target:, request:, options:) + @target = target + @request = request + @options = options + end + + def execute + if akismet.spam? + Gitlab::Recaptcha.enabled? ? REQUIRE_RECAPTCHA : DISALLOW + else + ALLOW + end + end + + private + + attr_reader :target, :request, :options + end +end diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index 1b9f5971f73..6bf04c55415 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -245,6 +245,34 @@ module SystemNoteService def auto_resolve_prometheus_alert(noteable, project, author) ::SystemNotes::IssuablesService.new(noteable: noteable, project: project, author: author).auto_resolve_prometheus_alert end + + # Parameters: + # - version [DesignManagement::Version] + # + # Example Note text: + # + # "added [1 designs](link-to-version)" + # "changed [2 designs](link-to-version)" + # + # Returns [Array<Note>]: the created Note objects + def design_version_added(version) + ::SystemNotes::DesignManagementService.new(noteable: version.issue, project: version.issue.project, author: version.author).design_version_added(version) + end + + # Called when a new discussion is created on a design + # + # discussion_note - DiscussionNote + # + # Example Note text: + # + # "started a discussion on screen.png" + # + # Returns the created Note object + def design_discussion_added(discussion_note) + design = discussion_note.noteable + + ::SystemNotes::DesignManagementService.new(noteable: design.issue, project: design.project, author: discussion_note.author).design_discussion_added(discussion_note) + end end SystemNoteService.prepend_if_ee('EE::SystemNoteService') diff --git a/app/services/system_notes/design_management_service.rb b/app/services/system_notes/design_management_service.rb new file mode 100644 index 00000000000..a773877e25b --- /dev/null +++ b/app/services/system_notes/design_management_service.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module SystemNotes + class DesignManagementService < ::SystemNotes::BaseService + include ActionView::RecordIdentifier + + # Parameters: + # - version [DesignManagement::Version] + # + # Example Note text: + # + # "added [1 designs](link-to-version)" + # "changed [2 designs](link-to-version)" + # + # Returns [Array<Note>]: the created Note objects + def design_version_added(version) + events = DesignManagement::Action.events + link_href = designs_path(version: version.id) + + version.designs_by_event.map do |(event_name, designs)| + note_data = self.class.design_event_note_data(events[event_name]) + icon_name = note_data[:icon] + n = designs.size + + body = "%s [%d %s](%s)" % [note_data[:past_tense], n, 'design'.pluralize(n), link_href] + + create_note(NoteSummary.new(noteable, project, author, body, action: icon_name)) + end + end + + # Called when a new discussion is created on a design + # + # discussion_note - DiscussionNote + # + # Example Note text: + # + # "started a discussion on screen.png" + # + # Returns the created Note object + def design_discussion_added(discussion_note) + design = discussion_note.noteable + + body = _('started a discussion on %{design_link}') % { + design_link: '[%s](%s)' % [ + design.filename, + designs_path(vueroute: design.filename, anchor: dom_id(discussion_note)) + ] + } + + action = :designs_discussion_added + + create_note(NoteSummary.new(noteable, project, author, body, action: action)) + end + + # Take one of the `DesignManagement::Action.events` and + # return: + # * an English past-tense verb. + # * the name of an icon used in renderin a system note + # + # We do not currently internationalize our system notes, + # instead we just produce English-language descriptions. + # See: https://gitlab.com/gitlab-org/gitlab/issues/30408 + # See: https://gitlab.com/gitlab-org/gitlab/issues/14056 + def self.design_event_note_data(event) + case event + when DesignManagement::Action.events[:creation] + { icon: 'designs_added', past_tense: 'added' } + when DesignManagement::Action.events[:modification] + { icon: 'designs_modified', past_tense: 'updated' } + when DesignManagement::Action.events[:deletion] + { icon: 'designs_removed', past_tense: 'removed' } + else + raise "Unknown event: #{event}" + end + end + + private + + def designs_path(params = {}) + url_helpers.designs_project_issue_path(project, noteable, params) + end + end +end diff --git a/app/services/tags/destroy_service.rb b/app/services/tags/destroy_service.rb index 4f6ae07be7d..3a01192487d 100644 --- a/app/services/tags/destroy_service.rb +++ b/app/services/tags/destroy_service.rb @@ -18,11 +18,6 @@ module Tags .new(project, current_user, tag: tag_name) .execute - push_data = build_push_data(tag) - EventCreateService.new.push(project, current_user, push_data) - project.execute_hooks(push_data.dup, :tag_push_hooks) - project.execute_services(push_data.dup, :tag_push_hooks) - success('Tag was removed') else error('Failed to remove tag') @@ -38,14 +33,5 @@ module Tags def success(message) super().merge(message: message) end - - def build_push_data(tag) - Gitlab::DataBuilder::Push.build( - project: project, - user: current_user, - oldrev: tag.dereferenced_target.sha, - newrev: Gitlab::Git::BLANK_SHA, - ref: "#{Gitlab::Git::TAG_REF_PREFIX}#{tag.name}") - end end end diff --git a/app/services/template_engines/liquid_service.rb b/app/services/template_engines/liquid_service.rb deleted file mode 100644 index 809ebd0316b..00000000000 --- a/app/services/template_engines/liquid_service.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module TemplateEngines - class LiquidService < BaseService - RenderError = Class.new(StandardError) - - DEFAULT_RENDER_SCORE_LIMIT = 1_000 - - def initialize(string) - @template = Liquid::Template.parse(string) - end - - def render(context, render_score_limit: DEFAULT_RENDER_SCORE_LIMIT) - set_limits(render_score_limit) - - @template.render!(context.stringify_keys) - rescue Liquid::MemoryError => e - handle_exception(e, string: @string, context: context) - - raise RenderError, _('Memory limit exceeded while rendering template') - rescue Liquid::Error => e - handle_exception(e, string: @string, context: context) - - raise RenderError, _('Error rendering query') - end - - private - - def set_limits(render_score_limit) - @template.resource_limits.render_score_limit = render_score_limit - - # We can also set assign_score_limit and render_length_limit if required. - - # render_score_limit limits the number of nodes (string, variable, block, tags) - # that are allowed in the template. - # render_length_limit seems to limit the sum of the bytesize of all node blocks. - # assign_score_limit seems to limit the sum of the bytesize of all capture blocks. - end - - def handle_exception(exception, extra = {}) - log_error(exception.message) - Gitlab::ErrorTracking.track_exception(exception, { - template_string: extra[:string], - variables: extra[:context] - }) - end - end -end diff --git a/app/services/terraform/remote_state_handler.rb b/app/services/terraform/remote_state_handler.rb index 5bb6f6a1dee..d180a3a2432 100644 --- a/app/services/terraform/remote_state_handler.rb +++ b/app/services/terraform/remote_state_handler.rb @@ -42,7 +42,7 @@ module Terraform state.lock_xid = params[:lock_id] state.locked_by_user = current_user - state.locked_at = Time.now + state.locked_at = Time.current state.save! end diff --git a/app/services/user_project_access_changed_service.rb b/app/services/user_project_access_changed_service.rb index 21d0861ac3f..66f1ccfab70 100644 --- a/app/services/user_project_access_changed_service.rb +++ b/app/services/user_project_access_changed_service.rb @@ -1,17 +1,26 @@ # frozen_string_literal: true class UserProjectAccessChangedService + DELAY = 1.hour + + HIGH_PRIORITY = :high + LOW_PRIORITY = :low + def initialize(user_ids) @user_ids = Array.wrap(user_ids) end - def execute(blocking: true) + def execute(blocking: true, priority: HIGH_PRIORITY) bulk_args = @user_ids.map { |id| [id] } if blocking AuthorizedProjectsWorker.bulk_perform_and_wait(bulk_args) else - AuthorizedProjectsWorker.bulk_perform_async(bulk_args) # rubocop:disable Scalability/BulkPerformWithContext + if priority == HIGH_PRIORITY + AuthorizedProjectsWorker.bulk_perform_async(bulk_args) # rubocop:disable Scalability/BulkPerformWithContext + else + AuthorizedProjectUpdate::UserRefreshWithLowUrgencyWorker.bulk_perform_in(DELAY, bulk_args) # rubocop:disable Scalability/BulkPerformWithContext + end end end end diff --git a/app/services/users/migrate_to_ghost_user_service.rb b/app/services/users/migrate_to_ghost_user_service.rb index e7186fdfb63..5ca9ed67e56 100644 --- a/app/services/users/migrate_to_ghost_user_service.rb +++ b/app/services/users/migrate_to_ghost_user_service.rb @@ -52,6 +52,7 @@ module Users migrate_notes migrate_abuse_reports migrate_award_emoji + migrate_snippets end # rubocop: disable CodeReuse/ActiveRecord @@ -79,6 +80,11 @@ module Users def migrate_award_emoji user.award_emoji.update_all(user_id: ghost_user.id) end + + def migrate_snippets + snippets = user.snippets.only_project_snippets + snippets.update_all(author_id: ghost_user.id) + end end end diff --git a/app/services/verify_pages_domain_service.rb b/app/services/verify_pages_domain_service.rb index b53c3145caf..a9e219547d7 100644 --- a/app/services/verify_pages_domain_service.rb +++ b/app/services/verify_pages_domain_service.rb @@ -37,7 +37,7 @@ class VerifyPagesDomainService < BaseService # Prevent any pre-existing grace period from being truncated reverify = [domain.enabled_until, VERIFICATION_PERIOD.from_now].compact.max - domain.assign_attributes(verified_at: Time.now, enabled_until: reverify, remove_at: nil) + domain.assign_attributes(verified_at: Time.current, enabled_until: reverify, remove_at: nil) domain.save!(validate: false) if was_disabled @@ -73,7 +73,7 @@ class VerifyPagesDomainService < BaseService # A domain is only expired until `disable!` has been called def expired? - domain.enabled_until && domain.enabled_until < Time.now + domain.enabled_until && domain.enabled_until < Time.current end def dns_record_present? diff --git a/app/services/wiki_pages/base_service.rb b/app/services/wiki_pages/base_service.rb index 2e774973ca5..a0256ea5e69 100644 --- a/app/services/wiki_pages/base_service.rb +++ b/app/services/wiki_pages/base_service.rb @@ -6,13 +6,13 @@ module WikiPages # - external_action: the action we report to external clients with webhooks # - usage_counter_action: the action that we count in out internal counters # - event_action: what we record as the value of `Event#action` - class BaseService < ::BaseService + class BaseService < ::BaseContainerService private def execute_hooks(page) page_data = payload(page) - @project.execute_hooks(page_data, :wiki_page_hooks) - @project.execute_services(page_data, :wiki_page_hooks) + container.execute_hooks(page_data, :wiki_page_hooks) + container.execute_services(page_data, :wiki_page_hooks) increment_usage create_wiki_event(page) end @@ -46,12 +46,9 @@ module WikiPages def create_wiki_event(page) return unless ::Feature.enabled?(:wiki_events) - slug = slug_for_page(page) + response = WikiPages::EventCreateService.new(current_user).execute(slug_for_page(page), page, event_action) - Event.transaction do - wiki_page_meta = WikiPage::Meta.find_or_create(slug, page) - EventCreateService.new.wiki_event(wiki_page_meta, current_user, event_action) - end + log_error(response.message) if response.error? end def slug_for_page(page) diff --git a/app/services/wiki_pages/create_service.rb b/app/services/wiki_pages/create_service.rb index 811f460e042..4ef19676d82 100644 --- a/app/services/wiki_pages/create_service.rb +++ b/app/services/wiki_pages/create_service.rb @@ -3,8 +3,8 @@ module WikiPages class CreateService < WikiPages::BaseService def execute - project_wiki = ProjectWiki.new(@project, current_user) - page = WikiPage.new(project_wiki) + wiki = Wiki.for_container(container, current_user) + page = WikiPage.new(wiki) if page.create(@params) execute_hooks(page) diff --git a/app/services/wiki_pages/event_create_service.rb b/app/services/wiki_pages/event_create_service.rb new file mode 100644 index 00000000000..18a45d057a9 --- /dev/null +++ b/app/services/wiki_pages/event_create_service.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module WikiPages + class EventCreateService + # @param [User] author The event author + def initialize(author) + raise ArgumentError, 'author must not be nil' unless author + + @author = author + end + + def execute(slug, page, action) + return ServiceResponse.success(message: 'No event created as `wiki_events` feature is disabled') unless ::Feature.enabled?(:wiki_events) + + event = Event.transaction do + wiki_page_meta = WikiPage::Meta.find_or_create(slug, page) + + ::EventCreateService.new.wiki_event(wiki_page_meta, author, action) + end + + ServiceResponse.success(payload: { event: event }) + rescue ::EventCreateService::IllegalActionError, ::ActiveRecord::ActiveRecordError => e + ServiceResponse.error(message: e.message, payload: { error: e }) + end + + private + + attr_reader :author + end +end diff --git a/app/services/wikis/create_attachment_service.rb b/app/services/wikis/create_attachment_service.rb index 6ef6cbc3c12..82179459345 100644 --- a/app/services/wikis/create_attachment_service.rb +++ b/app/services/wikis/create_attachment_service.rb @@ -5,12 +5,15 @@ module Wikis ATTACHMENT_PATH = 'uploads' MAX_FILENAME_LENGTH = 255 - delegate :wiki, to: :project + attr_reader :container + + delegate :wiki, to: :container delegate :repository, to: :wiki - def initialize(*args) - super + def initialize(container:, current_user: nil, params: {}) + super(nil, current_user, params) + @container = container @file_name = clean_file_name(params[:file_name]) @file_path = File.join(ATTACHMENT_PATH, SecureRandom.hex, @file_name) if @file_name @commit_message ||= "Upload attachment #{@file_name}" @@ -51,7 +54,7 @@ module Wikis end def validate_permissions! - unless can?(current_user, :create_wiki, project) + unless can?(current_user, :create_wiki, container) raise_error('You are not allowed to push to the wiki') end end |