diff options
Diffstat (limited to 'app/services')
37 files changed, 600 insertions, 254 deletions
diff --git a/app/services/award_emojis/add_service.rb b/app/services/award_emojis/add_service.rb new file mode 100644 index 00000000000..eac15dabbf0 --- /dev/null +++ b/app/services/award_emojis/add_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module AwardEmojis + class AddService < AwardEmojis::BaseService + include Gitlab::Utils::StrongMemoize + + def execute + unless awardable.user_can_award?(current_user) + return error('User cannot award emoji to awardable', status: :forbidden) + end + + unless awardable.emoji_awardable? + return error('Awardable cannot be awarded emoji', status: :unprocessable_entity) + end + + award = awardable.award_emoji.create(name: name, user: current_user) + + if award.persisted? + TodoService.new.new_award_emoji(todoable, current_user) if todoable + success(award: award) + else + error(award.errors.full_messages, award: award) + end + end + + private + + def todoable + strong_memoize(:todoable) do + case awardable + when Note + # We don't create todos for personal snippet comments for now + awardable.noteable unless awardable.for_personal_snippet? + when MergeRequest, Issue + awardable + when Snippet + nil + end + end + end + end +end diff --git a/app/services/award_emojis/base_service.rb b/app/services/award_emojis/base_service.rb new file mode 100644 index 00000000000..a677d03a221 --- /dev/null +++ b/app/services/award_emojis/base_service.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module AwardEmojis + class BaseService < ::BaseService + attr_accessor :awardable, :name + + def initialize(awardable, name, current_user) + @awardable = awardable + @name = normalize_name(name) + + super(awardable.project, current_user) + end + + private + + def normalize_name(name) + Gitlab::Emoji.normalize_emoji_name(name) + end + + # Provide more error state data than what BaseService allows. + # - An array of errors + # - The `AwardEmoji` if present + def error(errors, award: nil, status: nil) + errors = Array.wrap(errors) + + super(errors.to_sentence.presence, status).merge({ + award: award, + errors: errors + }) + end + end +end diff --git a/app/services/award_emojis/collect_user_emoji_service.rb b/app/services/award_emojis/collect_user_emoji_service.rb new file mode 100644 index 00000000000..6cab23f3edf --- /dev/null +++ b/app/services/award_emojis/collect_user_emoji_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Class for retrieving information about emoji awarded _by_ a particular user. +module AwardEmojis + class CollectUserEmojiService + attr_reader :current_user + + # current_user - The User to generate the data for. + def initialize(current_user = nil) + @current_user = current_user + end + + def execute + return [] unless current_user + + # We want the resulting data set to be an Array containing the emoji names + # in descending order, based on how often they were awarded. + AwardEmoji + .award_counts_for_user(current_user) + .map { |name, _| { name: name } } + end + end +end diff --git a/app/services/award_emojis/destroy_service.rb b/app/services/award_emojis/destroy_service.rb new file mode 100644 index 00000000000..3789a8403bc --- /dev/null +++ b/app/services/award_emojis/destroy_service.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module AwardEmojis + class DestroyService < AwardEmojis::BaseService + def execute + unless awardable.user_can_award?(current_user) + return error('User cannot destroy emoji on the awardable', status: :forbidden) + end + + awards = AwardEmojisFinder.new(awardable, name: name, awarded_by: current_user).execute + + if awards.empty? + return error("User has not awarded emoji of type #{name} on the awardable", status: :forbidden) + end + + award = awards.destroy_all.first # rubocop: disable DestroyAll + + success(award: award) + end + end +end diff --git a/app/services/award_emojis/toggle_service.rb b/app/services/award_emojis/toggle_service.rb new file mode 100644 index 00000000000..812dd1c2889 --- /dev/null +++ b/app/services/award_emojis/toggle_service.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module AwardEmojis + class ToggleService < AwardEmojis::BaseService + def execute + if awardable.awarded_emoji?(name, current_user) + DestroyService.new(awardable, name, current_user).execute + else + AddService.new(awardable, name, current_user).execute + end + end + end +end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index cdcc4b15bea..29317f1176e 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -15,7 +15,8 @@ module Ci Gitlab::Ci::Pipeline::Chain::Limit::Size, Gitlab::Ci::Pipeline::Chain::Populate, Gitlab::Ci::Pipeline::Chain::Create, - Gitlab::Ci::Pipeline::Chain::Limit::Activity].freeze + Gitlab::Ci::Pipeline::Chain::Limit::Activity, + Gitlab::Ci::Pipeline::Chain::Limit::JobActivity].freeze def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, **options, &block) @pipeline = Ci::Pipeline.new diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 99d4ff9ecd1..3b145a65d79 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -40,16 +40,21 @@ module Ci def process_builds_with_needs(trigger_build_ids) return false unless trigger_build_ids.present? - return false unless Feature.enabled?(:ci_dag_support, project) + return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true) - # rubocop: disable CodeReuse/ActiveRecord - trigger_build_names = pipeline.statuses - .where(id: trigger_build_ids) - .select(:name) - # rubocop: enable CodeReuse/ActiveRecord + # we find processables that are dependent: + # 1. because of current dependency, + trigger_build_names = pipeline.processables.latest + .for_ids(trigger_build_ids).names + # 2. does not have builds that not yet complete + incomplete_build_names = pipeline.processables.latest + .incomplete.names + + # Each found processable is guaranteed here to have completed status created_processables .with_needs(trigger_build_names) + .without_needs(incomplete_build_names) .find_each .map(&method(:process_build_with_needs)) .any? @@ -70,17 +75,13 @@ module Ci end end - # rubocop: disable CodeReuse/ActiveRecord def status_for_prior_stages(index) - pipeline.builds.where('stage_idx < ?', index).latest.status || 'success' + pipeline.processables.status_for_prior_stages(index) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def status_for_build_needs(needs) - pipeline.builds.where(name: needs).latest.status || 'success' + pipeline.processables.status_for_names(needs) end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def stage_indexes_of_created_processables_without_needs @@ -89,15 +90,13 @@ module Ci end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def created_processables_in_stage_without_needs(index) created_processables_without_needs - .where(stage_idx: index) + .for_stage(index) end - # rubocop: enable CodeReuse/ActiveRecord def created_processables_without_needs - if Feature.enabled?(:ci_dag_support, project) + if Feature.enabled?(:ci_dag_support, project, default_enabled: true) pipeline.processables.created.without_needs else pipeline.processables.created diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb index 9c589d910eb..31c7178c9e7 100644 --- a/app/services/ci/update_build_queue_service.rb +++ b/app/services/ci/update_build_queue_service.rb @@ -9,6 +9,10 @@ module Ci private def tick_for(build, runners) + if Feature.enabled?(:ci_update_queues_for_online_runners, build.project, default_enabled: true) + runners = runners.with_recent_runner_queue + end + runners.each do |runner| runner.pick_build!(build) end diff --git a/app/services/clusters/build_kubernetes_namespace_service.rb b/app/services/clusters/build_kubernetes_namespace_service.rb new file mode 100644 index 00000000000..2574f77bbf9 --- /dev/null +++ b/app/services/clusters/build_kubernetes_namespace_service.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Clusters + class BuildKubernetesNamespaceService + attr_reader :cluster, :environment + + def initialize(cluster, environment:) + @cluster = cluster + @environment = environment + end + + def execute + cluster.kubernetes_namespaces.build(attributes) + end + + private + + def attributes + attributes = { + project: environment.project, + namespace: namespace, + service_account_name: "#{namespace}-service-account" + } + + attributes[:cluster_project] = cluster.cluster_project if cluster.project_type? + attributes[:environment] = environment if cluster.namespace_per_environment? + + attributes + end + + def namespace + Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: environment.project).from_environment_slug(environment.slug) + end + end +end diff --git a/app/services/clusters/create_service.rb b/app/services/clusters/create_service.rb index 5fb5e15c32d..e5a5b73321a 100644 --- a/app/services/clusters/create_service.rb +++ b/app/services/clusters/create_service.rb @@ -11,7 +11,8 @@ module Clusters def execute(access_token: nil) raise ArgumentError, 'Unknown clusterable provided' unless clusterable - cluster_params = params.merge(user: current_user).merge(clusterable_params) + cluster_params = params.merge(global_params).merge(clusterable_params) + cluster_params[:provider_gcp_attributes].try do |provider| provider[:access_token] = access_token end @@ -35,6 +36,10 @@ module Clusters @clusterable ||= params.delete(:clusterable) end + def global_params + { user: current_user, namespace_per_environment: Feature.enabled?(:kubernetes_namespace_per_environment, default_enabled: true) } + end + def clusterable_params case clusterable when ::Project diff --git a/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb b/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb index 806f320381d..c45dac7b273 100644 --- a/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb +++ b/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb @@ -11,7 +11,6 @@ module Clusters end def execute - configure_kubernetes_namespace create_project_service_account configure_kubernetes_token @@ -22,10 +21,6 @@ module Clusters attr_reader :cluster, :kubernetes_namespace, :platform - def configure_kubernetes_namespace - kubernetes_namespace.set_defaults - end - def create_project_service_account Clusters::Gcp::Kubernetes::CreateOrUpdateServiceAccountService.namespace_creator( platform.kubeclient, diff --git a/app/services/create_snippet_service.rb b/app/services/create_snippet_service.rb index 6f1fce4989e..6e5bf823cc7 100644 --- a/app/services/create_snippet_service.rb +++ b/app/services/create_snippet_service.rb @@ -23,6 +23,7 @@ class CreateSnippetService < BaseService if snippet.save UserAgentDetailService.new(snippet, @request).create + Gitlab::UsageDataCounters::SnippetCounter.count(:create) end snippet diff --git a/app/services/git/base_hooks_service.rb b/app/services/git/base_hooks_service.rb index d30df34e54b..47c308c8280 100644 --- a/app/services/git/base_hooks_service.rb +++ b/app/services/git/base_hooks_service.rb @@ -8,8 +8,6 @@ module Git PROCESS_COMMIT_LIMIT = 100 def execute - project.repository.after_create if project.empty_repo? - create_events create_pipelines execute_project_hooks @@ -19,7 +17,7 @@ module Git update_remote_mirrors - push_data + success end private @@ -33,7 +31,7 @@ module Git end def limited_commits - commits.last(PROCESS_COMMIT_LIMIT) + @limited_commits ||= commits.last(PROCESS_COMMIT_LIMIT) end def commits_count @@ -48,43 +46,69 @@ module Git [] end + # Push events in the activity feed only show information for the + # last commit. def create_events - EventCreateService.new.push(project, current_user, push_data) + EventCreateService.new.push(project, current_user, event_push_data) end def create_pipelines return unless params.fetch(:create_pipelines, true) Ci::CreatePipelineService - .new(project, current_user, push_data) + .new(project, current_user, pipeline_params) .execute(:push, pipeline_options) end def execute_project_hooks - project.execute_hooks(push_data, hook_name) - project.execute_services(push_data, hook_name) + # Creating push_data invokes one CommitDelta RPC per commit. Only + # build this data if we actually need it. + project.execute_hooks(push_data, hook_name) if project.has_active_hooks?(hook_name) + project.execute_services(push_data, hook_name) if project.has_active_services?(hook_name) end def enqueue_invalidate_cache - ProjectCacheWorker.perform_async( - project.id, - invalidated_file_types, - [:commit_count, :repository_size] - ) + file_types = invalidated_file_types + + return unless file_types.present? + + ProjectCacheWorker.perform_async(project.id, file_types, [], false) end - def push_data - @push_data ||= Gitlab::DataBuilder::Push.build( - project: project, - user: current_user, + def pipeline_params + { + before: params[:oldrev], + after: params[:newrev], + ref: params[:ref], + push_options: params[:push_options] || {}, + checkout_sha: Gitlab::DataBuilder::Push.checkout_sha( + project.repository, params[:newrev], params[:ref]) + } + end + + def push_data_params(commits:, with_changed_files: true) + { oldrev: params[:oldrev], newrev: params[:newrev], ref: params[:ref], - commits: limited_commits, + project: project, + user: current_user, + commits: commits, message: event_message, commits_count: commits_count, - push_options: params[:push_options] || {} - ) + with_changed_files: with_changed_files + } + end + + def event_push_data + # We only need the last commit for the event push, and we don't + # need the full deltas either. + @event_push_data ||= Gitlab::DataBuilder::Push.build( + push_data_params(commits: commits.last, with_changed_files: false)) + end + + def push_data + @push_data ||= Gitlab::DataBuilder::Push.build(push_data_params(commits: limited_commits)) # Dependent code may modify the push data, so return a duplicate each time @push_data.dup diff --git a/app/services/git/branch_hooks_service.rb b/app/services/git/branch_hooks_service.rb index c41f445c3c4..d2b037a680c 100644 --- a/app/services/git/branch_hooks_service.rb +++ b/app/services/git/branch_hooks_service.rb @@ -63,7 +63,7 @@ module Git end def branch_create_hooks - project.repository.after_create_branch + project.repository.after_create_branch(expire_cache: false) project.after_create_default_branch if default_branch? end @@ -78,13 +78,21 @@ module Git end def branch_remove_hooks - project.repository.after_remove_branch + project.repository.after_remove_branch(expire_cache: false) end # Schedules processing of commit messages def enqueue_process_commit_messages - limited_commits.each do |commit| - next unless commit.matches_cross_reference_regex? + referencing_commits = limited_commits.select(&:matches_cross_reference_regex?) + + upstream_commit_ids = upstream_commit_ids(referencing_commits) + + referencing_commits.each do |commit| + # Avoid reprocessing commits that already exist upstream if the project + # is a fork. This will prevent duplicated/superfluous system notes on + # mentionables referenced by a commit that is pushed to the upstream, + # that is then also pushed to forks when these get synced by users. + next if upstream_commit_ids.include?(commit.id) ProcessCommitWorker.perform_async( project.id, @@ -142,5 +150,18 @@ module Git def branch_name strong_memoize(:branch_name) { Gitlab::Git.ref_name(params[:ref]) } end + + def upstream_commit_ids(commits) + set = Set.new + + upstream_project = project.fork_source + if upstream_project + upstream_project + .commits_by(oids: commits.map(&:id)) + .each { |commit| set << commit.id } + end + + set + end end end diff --git a/app/services/groups/update_service.rb b/app/services/groups/update_service.rb index 73e1e00dc33..116756bacfe 100644 --- a/app/services/groups/update_service.rb +++ b/app/services/groups/update_service.rb @@ -46,6 +46,11 @@ module Groups params.delete(:parent_id) end + # overridden in EE + def remove_unallowed_params + params.delete(:emails_disabled) unless can?(current_user, :set_emails_disabled, group) + end + def valid_share_with_group_lock_change? return true unless changing_share_with_group_lock? return true if can?(current_user, :change_share_with_group_lock, group) diff --git a/app/services/issuable/bulk_update_service.rb b/app/services/issuable/bulk_update_service.rb index 6d215d7a3b9..273a12f386a 100644 --- a/app/services/issuable/bulk_update_service.rb +++ b/app/services/issuable/bulk_update_service.rb @@ -29,7 +29,7 @@ module Issuable items.each do |issuable| next unless can?(current_user, :"update_#{type}", issuable) - update_class.new(issuable.project, current_user, params).execute(issuable) + update_class.new(issuable.issuing_parent, current_user, params).execute(issuable) end { diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 77c2224ee3b..2ab6e88599f 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -344,10 +344,7 @@ class IssuableBaseService < BaseService def toggle_award(issuable) award = params.delete(:emoji_award) - if award - todo_service.new_award_emoji(issuable, current_user) - issuable.toggle_award_emoji(award, current_user) - end + AwardEmojis::ToggleService.new(issuable, award, current_user).execute if award end def associations_before_update(issuable) diff --git a/app/services/members/base_service.rb b/app/services/members/base_service.rb index e78affff797..5d69418fb7d 100644 --- a/app/services/members/base_service.rb +++ b/app/services/members/base_service.rb @@ -51,7 +51,9 @@ module Members def enqueue_delete_todos(member) type = member.is_a?(GroupMember) ? 'Group' : 'Project' # don't enqueue immediately to prevent todos removal in case of a mistake - TodosDestroyer::EntityLeaveWorker.perform_in(Todo::WAIT_FOR_DELETE, member.user_id, member.source_id, type) + member.run_after_commit_or_now do + TodosDestroyer::EntityLeaveWorker.perform_in(Todo::WAIT_FOR_DELETE, member.user_id, member.source_id, type) + end end end end diff --git a/app/services/merge_requests/base_service.rb b/app/services/merge_requests/base_service.rb index 067510a8a0a..c6aae4c28f2 100644 --- a/app/services/merge_requests/base_service.rb +++ b/app/services/merge_requests/base_service.rb @@ -17,11 +17,9 @@ module MergeRequests end def execute_hooks(merge_request, action = 'open', old_rev: nil, old_associations: {}) - if merge_request.project - merge_data = hook_data(merge_request, action, old_rev: old_rev, old_associations: old_associations) - merge_request.project.execute_hooks(merge_data, :merge_request_hooks) - merge_request.project.execute_services(merge_data, :merge_request_hooks) - end + merge_data = hook_data(merge_request, action, old_rev: old_rev, old_associations: old_associations) + merge_request.project.execute_hooks(merge_data, :merge_request_hooks) + merge_request.project.execute_services(merge_data, :merge_request_hooks) end def cleanup_environments(merge_request) diff --git a/app/services/merge_requests/rebase_service.rb b/app/services/merge_requests/rebase_service.rb index 8d3b9b05819..27c16ba1777 100644 --- a/app/services/merge_requests/rebase_service.rb +++ b/app/services/merge_requests/rebase_service.rb @@ -15,7 +15,8 @@ module MergeRequests end def rebase - if merge_request.gitaly_rebase_in_progress? + # Ensure Gitaly isn't already running a rebase + if source_project.repository.rebase_in_progress?(merge_request.id) log_error('Rebase task canceled: Another rebase is already in progress', save_message_on_model: true) return false end diff --git a/app/services/metrics/dashboard/base_embed_service.rb b/app/services/metrics/dashboard/base_embed_service.rb new file mode 100644 index 00000000000..8bb5f4892cb --- /dev/null +++ b/app/services/metrics/dashboard/base_embed_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Base class for embed services. Contains a few basic helper +# methods that the embed services share. +module Metrics + module Dashboard + class BaseEmbedService < ::Metrics::Dashboard::BaseService + def cache_key + "dynamic_metrics_dashboard_#{identifiers}" + end + + protected + + def dashboard_path + params[:dashboard_path].presence || + ::Metrics::Dashboard::SystemDashboardService::SYSTEM_DASHBOARD_PATH + end + + def group + params[:group] + end + + def title + params[:title] + end + + def y_label + params[:y_label] + end + + def identifiers + [dashboard_path, group, title, y_label].join('|') + end + end + end +end diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb index b331bf51874..8a42675c66d 100644 --- a/app/services/metrics/dashboard/base_service.rb +++ b/app/services/metrics/dashboard/base_service.rb @@ -5,17 +5,14 @@ module Metrics module Dashboard class BaseService < ::BaseService - PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError + include Gitlab::Metrics::Dashboard::Errors def get_dashboard return error('Insufficient permissions.', :unauthorized) unless allowed? success(dashboard: process_dashboard) - rescue NOT_FOUND_ERROR - error("#{dashboard_path} could not be found.", :not_found) - rescue PROCESSING_ERROR => e - error(e.message, :unprocessable_entity) + rescue StandardError => e + handle_errors(e) end # Summary of all known dashboards for the service. diff --git a/app/services/metrics/dashboard/custom_metric_embed_service.rb b/app/services/metrics/dashboard/custom_metric_embed_service.rb new file mode 100644 index 00000000000..50f070989fc --- /dev/null +++ b/app/services/metrics/dashboard/custom_metric_embed_service.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +# Responsible for returning a dashboard containing specified +# custom metrics. Creates panels based on the matching metrics +# stored in the database. +# +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class CustomMetricEmbedService < ::Metrics::Dashboard::BaseEmbedService + extend ::Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize + include Gitlab::Metrics::Dashboard::Defaults + + class << self + # Determines whether the provided params are sufficient + # to uniquely identify a panel composed of user-defined + # custom metrics from the DB. + def valid_params?(params) + [ + params[:embedded], + valid_dashboard?(params[:dashboard_path]), + valid_group_title?(params[:group]), + params[:title].present?, + params.has_key?(:y_label) + ].all? + end + + private + + # A group title is valid if it is one of the limited + # options the user can select in the UI. + def valid_group_title?(group) + PrometheusMetricEnums + .custom_group_details + .map { |_, details| details[:group_title] } + .include?(group) + end + + # All custom metrics are displayed on the system dashboard. + # Nil is acceptable as we'll default to the system dashboard. + def valid_dashboard?(dashboard) + dashboard.nil? || SystemDashboardService.system_dashboard?(dashboard) + end + end + + # Returns a new dashboard with only the matching + # metrics from the system dashboard, stripped of + # group info. + # + # Note: This overrides the method #raw_dashboard, + # which means the result will not be cached. This + # is because we are inserting DB info into the + # dashboard before post-processing. This ensures + # we aren't acting on deleted or out-of-date metrics. + # + # @return [Hash] + override :raw_dashboard + def raw_dashboard + panels_not_found!(identifiers) if panels.empty? + + { 'panel_groups' => [{ 'panels' => panels }] } + end + + private + + # Generated dashboard panels for each metric which + # matches the provided input. + # @return [Array<Hash>] + def panels + strong_memoize(:panels) do + metrics.map { |metric| panel_for_metric(metric) } + end + end + + # Metrics which match the provided inputs. + # There may be multiple metrics, but they should be + # displayed in a single panel/chart. + # @return [ActiveRecord::AssociationRelation<PromtheusMetric>] + # rubocop: disable CodeReuse/ActiveRecord + def metrics + project.prometheus_metrics.where( + group: group_key, + title: title, + y_label: y_label + ) + end + # rubocop: enable CodeReuse/ActiveRecord + + # Returns a symbol representing the group that + # the dashboard's group title belongs to. + # It will be one of the keys found under + # PrometheusMetricEnums.custom_groups. + # + # @return [String] + def group_key + strong_memoize(:group_key) do + PrometheusMetricEnums + .group_details + .find { |_, details| details[:group_title] == group } + .first + .to_s + end + end + + # Returns a representation of a PromtheusMetric + # as a dashboard panel. As the panel is generated + # on the fly, we're using default values for info + # not represented in the DB. + # + # @return [Hash] + def panel_for_metric(metric) + { + type: DEFAULT_PANEL_TYPE, + weight: DEFAULT_PANEL_WEIGHT, + title: metric.title, + y_label: metric.y_label, + metrics: [metric.to_metric_hash] + } + end + end + end +end diff --git a/app/services/metrics/dashboard/default_embed_service.rb b/app/services/metrics/dashboard/default_embed_service.rb index 8b01b44fc98..e1bd98bd5c2 100644 --- a/app/services/metrics/dashboard/default_embed_service.rb +++ b/app/services/metrics/dashboard/default_embed_service.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true # Responsible for returning a filtered system dashboard -# containing only the default embedded metrics. In future, -# this class may be updated to support filtering to -# alternate metrics/panels. +# containing only the default embedded metrics. This class +# operates by selecting metrics directly from the system +# dashboard. # # Why isn't this filtering in a processing stage? By filtering # here, we ensure the dynamically-determined dashboard is cached. @@ -11,7 +11,7 @@ # Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. module Metrics module Dashboard - class DefaultEmbedService < ::Metrics::Dashboard::BaseService + class DefaultEmbedService < ::Metrics::Dashboard::BaseEmbedService # For the default filtering for embedded metrics, # uses the 'id' key in dashboard-yml definition for # identification. @@ -33,10 +33,6 @@ module Metrics { 'panel_groups' => [{ 'panels' => panels }] } end - def cache_key - "dynamic_metrics_dashboard_#{metric_identifiers.join('_')}" - end - private # Returns an array of the panels groups on the @@ -58,6 +54,10 @@ module Metrics def metric_identifiers DEFAULT_EMBEDDED_METRICS_IDENTIFIERS end + + def identifiers + metric_identifiers.join('|') + end end end end diff --git a/app/services/metrics/dashboard/dynamic_embed_service.rb b/app/services/metrics/dashboard/dynamic_embed_service.rb new file mode 100644 index 00000000000..db5b7c9e32a --- /dev/null +++ b/app/services/metrics/dashboard/dynamic_embed_service.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# Responsible for returning a filtered project dashboard +# containing only the request-provided metrics. The result +# is then cached for future requests. Metrics are identified +# based on a combination of identifiers for now, but the ideal +# would be similar to the approach in DefaultEmbedService, but +# a single unique identifier is not currently available across +# all metric types (custom, project-defined, cluster, or system). +# +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class DynamicEmbedService < ::Metrics::Dashboard::BaseEmbedService + include Gitlab::Utils::StrongMemoize + + class << self + # Determines whether the provided params are sufficient + # to uniquely identify a panel from a yml-defined dashboard. + # + # See https://docs.gitlab.com/ee/user/project/integrations/prometheus.html#defining-custom-dashboards-per-project + # for additional info on defining custom dashboards. + def valid_params?(params) + [ + params[:embedded], + params[:group].present?, + params[:title].present?, + params[:y_label] + ].all? + end + end + + # Returns a new dashboard with only the matching + # metrics from the system dashboard, stripped of groups. + # @return [Hash] + def get_raw_dashboard + not_found! if panels.empty? + + { 'panel_groups' => [{ 'panels' => panels }] } + end + + private + + def panels + strong_memoize(:panels) do + not_found! unless base_dashboard + not_found! unless groups = base_dashboard['panel_groups'] + not_found! unless matching_group = find_group(groups) + not_found! unless all_panels = matching_group['panels'] + + find_panels(all_panels) + end + end + + def base_dashboard + strong_memoize(:base_dashboard) do + Gitlab::Metrics::Dashboard::Finder.find_raw(project, dashboard_path: dashboard_path) + end + end + + def find_group(groups) + groups.find do |candidate_group| + candidate_group['group'] == group + end + end + + def find_panels(all_panels) + all_panels.select do |panel| + panel['title'] == title && panel['y_label'] == y_label + end + end + + def not_found! + panels_not_found!(identifiers) + end + end + end +end diff --git a/app/services/notes/base_service.rb b/app/services/notes/base_service.rb index c1260837c12..b4d04c47cc0 100644 --- a/app/services/notes/base_service.rb +++ b/app/services/notes/base_service.rb @@ -9,5 +9,9 @@ module Notes note.noteable.diffs.clear_cache end end + + def increment_usage_counter(note) + Gitlab::UsageDataCounters::NoteCounter.count(:create, note.noteable_type) + end end end diff --git a/app/services/notes/create_service.rb b/app/services/notes/create_service.rb index 194c4a43dbc..248e81080cc 100644 --- a/app/services/notes/create_service.rb +++ b/app/services/notes/create_service.rb @@ -41,6 +41,7 @@ module Notes todo_service.new_note(note, current_user) clear_noteable_diffs_cache(note) Suggestions::CreateService.new(note).execute + increment_usage_counter(note) end if quick_actions_service.commands_executed_count.to_i > 0 @@ -52,7 +53,7 @@ module Notes # We must add the error after we call #save because errors are reset # when #save is called if only_commands - note.errors.add(:commands_only, message.presence || _('Commands did not apply')) + note.errors.add(:commands_only, message.presence || _('Failed to apply commands.')) end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 21fab22e0d4..83710ffce2f 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -321,6 +321,9 @@ class NotificationService end def decline_project_invite(project_member) + # Must always send, regardless of project/namespace configuration since it's a + # response to the user's action. + mailer.member_invite_declined_email( project_member.real_source_type, project_member.project.id, @@ -351,8 +354,8 @@ class NotificationService end def decline_group_invite(group_member) - # always send this one, since it's a response to the user's own - # action + # Must always send, regardless of project/namespace configuration since it's a + # response to the user's action. mailer.member_invite_declined_email( group_member.real_source_type, @@ -410,6 +413,10 @@ class NotificationService end def pipeline_finished(pipeline, recipients = nil) + # Must always check project configuration since recipients could be a list of emails + # from the PipelinesEmailService integration. + return if pipeline.project.emails_disabled? + email_template = "pipeline_#{pipeline.status}_email" return unless mailer.respond_to?(email_template) @@ -428,6 +435,8 @@ class NotificationService end def autodevops_disabled(pipeline, recipients) + return if pipeline.project.emails_disabled? + recipients.each do |recipient| mailer.autodevops_disabled_email(pipeline, recipient).deliver_later end @@ -472,10 +481,14 @@ class NotificationService end def repository_cleanup_success(project, user) + return if project.emails_disabled? + mailer.send(:repository_cleanup_success_email, project, user).deliver_later end def repository_cleanup_failure(project, user, error) + return if project.emails_disabled? + mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index a1279bfb3a3..5893b8eedff 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -173,6 +173,7 @@ module Projects end def remove_registry_tags + return true unless Gitlab.config.registry.enabled return false unless remove_legacy_registry_tags project.container_repositories.find_each do |container_repository| diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 1244a0f72a7..13a467a3ef9 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -2,31 +2,52 @@ module Projects class UpdateRemoteMirrorService < BaseService - attr_reader :errors + MAX_TRIES = 3 - def execute(remote_mirror) + def execute(remote_mirror, tries) return success unless remote_mirror.enabled? - errors = [] + update_mirror(remote_mirror) - begin - remote_mirror.ensure_remote! - repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true) + success + rescue Gitlab::Git::CommandError => e + # This happens if one of the gitaly calls above fail, for example when + # branches have diverged, or the pre-receive hook fails. + retry_or_fail(remote_mirror, e.message, tries) - opts = {} - if remote_mirror.only_protected_branches? - opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) - end + error(e.message) + rescue => e + remote_mirror.mark_as_failed!(e.message) + raise e + end + + private + + def update_mirror(remote_mirror) + remote_mirror.update_start! + + remote_mirror.ensure_remote! + repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true) - remote_mirror.update_repository(opts) - rescue => e - errors << e.message.strip + opts = {} + if remote_mirror.only_protected_branches? + opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) end - if errors.present? - error(errors.join("\n\n")) + remote_mirror.update_repository(opts) + + remote_mirror.update_finish! + end + + def retry_or_fail(mirror, message, tries) + if tries < MAX_TRIES + mirror.mark_for_retry!(message) else - success + # It's not likely we'll be able to recover from this ourselves, so we'll + # notify the users of the problem, and don't trigger any sidekiq retries + # Instead, we'll wait for the next change to try the push again, or until + # a user manually retries. + mirror.mark_as_failed!(message) end end end diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index 2bc04470342..8acbdc7e02b 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -9,6 +9,7 @@ module Projects # rubocop: disable CodeReuse/ActiveRecord def execute + remove_unallowed_params validate! ensure_wiki_exists if enabling_wiki? @@ -54,6 +55,10 @@ module Projects end end + def remove_unallowed_params + params.delete(:emails_disabled) unless can?(current_user, :set_emails_disabled, project) + end + def after_update todos_features_changes = %w( issues_access_level @@ -122,7 +127,7 @@ module Projects ProjectWiki.new(project, project.owner).wiki rescue ProjectWiki::CouldNotCreateWikiError log_error("Could not create wiki for #{project.full_name}") - Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki') + Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki').increment end def update_pages_config diff --git a/app/services/prometheus/proxy_service.rb b/app/services/prometheus/proxy_service.rb index c5d2b84878b..a62eb76b8ce 100644 --- a/app/services/prometheus/proxy_service.rb +++ b/app/services/prometheus/proxy_service.rb @@ -98,7 +98,7 @@ module Prometheus end def prometheus_client_wrapper - prometheus_adapter&.prometheus_client_wrapper + prometheus_adapter&.prometheus_client end def can_query? diff --git a/app/services/self_monitoring/project/create_service.rb b/app/services/self_monitoring/project/create_service.rb deleted file mode 100644 index 8ffd22de127..00000000000 --- a/app/services/self_monitoring/project/create_service.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: true - -module SelfMonitoring - module Project - class CreateService < ::BaseService - include Stepable - - DEFAULT_VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL - DEFAULT_NAME = 'GitLab Instance Administration' - DEFAULT_DESCRIPTION = <<~HEREDOC - This project is automatically generated and will be used to help monitor this GitLab instance. - HEREDOC - - steps :validate_admins, - :create_project, - :add_project_members, - :add_to_whitelist, - :add_prometheus_manual_configuration - - def initialize - super(nil) - end - - def execute - execute_steps - end - - private - - def validate_admins - unless instance_admins.any? - log_error('No active admin user found') - return error('No active admin user found') - end - - success - end - - def create_project - admin_user = project_owner - @project = ::Projects::CreateService.new(admin_user, create_project_params).execute - - if project.persisted? - success(project: project) - else - log_error("Could not create self-monitoring project. Errors: #{project.errors.full_messages}") - error('Could not create project') - end - end - - def add_project_members - members = project.add_users(project_maintainers, Gitlab::Access::MAINTAINER) - errors = members.flat_map { |member| member.errors.full_messages } - - if errors.any? - log_error("Could not add admins as members to self-monitoring project. Errors: #{errors}") - error('Could not add admins as members') - else - success - end - end - - def add_to_whitelist - return success unless prometheus_enabled? - return success unless prometheus_listen_address.present? - - uri = parse_url(internal_prometheus_listen_address_uri) - return error(_('Prometheus listen_address is not a valid URI')) unless uri - - result = ApplicationSettings::UpdateService.new( - Gitlab::CurrentSettings.current_application_settings, - project_owner, - outbound_local_requests_whitelist: [uri.normalized_host] - ).execute - - if result - success - else - error(_('Could not add prometheus URL to whitelist')) - end - end - - def add_prometheus_manual_configuration - return success unless prometheus_enabled? - return success unless prometheus_listen_address.present? - - service = project.find_or_initialize_service('prometheus') - - unless service.update(prometheus_service_attributes) - log_error("Could not save prometheus manual configuration for self-monitoring project. Errors: #{service.errors.full_messages}") - return error('Could not save prometheus manual configuration') - end - - success - end - - def parse_url(uri_string) - Addressable::URI.parse(uri_string) - rescue Addressable::URI::InvalidURIError, TypeError - end - - def prometheus_enabled? - Gitlab.config.prometheus.enable - rescue Settingslogic::MissingSetting - false - end - - def prometheus_listen_address - Gitlab.config.prometheus.listen_address - rescue Settingslogic::MissingSetting - end - - def instance_admins - @instance_admins ||= User.admins.active - end - - def project_owner - instance_admins.first - end - - def project_maintainers - # Exclude the first so that the project_owner is not added again as a member. - instance_admins - [project_owner] - end - - def create_project_params - { - initialize_with_readme: true, - visibility_level: DEFAULT_VISIBILITY_LEVEL, - name: DEFAULT_NAME, - description: DEFAULT_DESCRIPTION - } - end - - def internal_prometheus_listen_address_uri - if prometheus_listen_address.starts_with?('http') - prometheus_listen_address - else - 'http://' + prometheus_listen_address - end - end - - def prometheus_service_attributes - { - api_url: internal_prometheus_listen_address_uri, - manual_configuration: true, - active: true - } - end - end - end -end diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index e30debbbe75..ee7223d6349 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -598,11 +598,11 @@ module SystemNoteService end def zoom_link_added(issue, project, author) - create_note(NoteSummary.new(issue, project, author, _('a Zoom call was added to this issue'), action: 'pinned_embed')) + create_note(NoteSummary.new(issue, project, author, _('added a Zoom call to this issue'), action: 'pinned_embed')) end def zoom_link_removed(issue, project, author) - create_note(NoteSummary.new(issue, project, author, _('a Zoom call was removed from this issue'), action: 'pinned_embed')) + create_note(NoteSummary.new(issue, project, author, _('removed a Zoom call from this issue'), action: 'pinned_embed')) end private diff --git a/app/services/todo_service.rb b/app/services/todo_service.rb index 0ea230a44a1..b1256df35d6 100644 --- a/app/services/todo_service.rb +++ b/app/services/todo_service.rb @@ -314,11 +314,9 @@ class TodoService end def reject_users_without_access(users, parent, target) - if target.is_a?(Note) && target.for_issuable? - target = target.noteable - end + target = target.noteable if target.is_a?(Note) - if target.is_a?(Issuable) + if target.respond_to?(:to_ability_name) select_users(users, :"read_#{target.to_ability_name}", target) else select_users(users, :read_project, parent) diff --git a/app/services/update_deployment_service.rb b/app/services/update_deployment_service.rb index 49a7d0178f4..dcafebae52d 100644 --- a/app/services/update_deployment_service.rb +++ b/app/services/update_deployment_service.rb @@ -42,7 +42,7 @@ class UpdateDeploymentService return unless environment_url @expanded_environment_url = - ExpandVariables.expand(environment_url, variables) + ExpandVariables.expand(environment_url, -> { variables }) end def environment_url diff --git a/app/services/update_snippet_service.rb b/app/services/update_snippet_service.rb index 15bc1046a4e..2969c360de5 100644 --- a/app/services/update_snippet_service.rb +++ b/app/services/update_snippet_service.rb @@ -25,6 +25,8 @@ class UpdateSnippetService < BaseService snippet.assign_attributes(params) spam_check(snippet, current_user) - snippet.save + snippet.save.tap do |succeeded| + Gitlab::UsageDataCounters::SnippetCounter.count(:update) if succeeded + end end end |