diff options
Diffstat (limited to 'app/services')
89 files changed, 1332 insertions, 275 deletions
diff --git a/app/services/akismet_service.rb b/app/services/akismet_service.rb index 82ae66ab0f5..63be3c371ec 100644 --- a/app/services/akismet_service.rb +++ b/app/services/akismet_service.rb @@ -25,7 +25,7 @@ class AkismetService is_spam, is_blatant = akismet_client.check(options[:ip_address], options[:user_agent], params) is_spam || is_blatant rescue => e - Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") # rubocop:disable Gitlab/RailsLogger false end end @@ -63,7 +63,7 @@ class AkismetService akismet_client.public_send(type, options[:ip_address], options[:user_agent], params) # rubocop:disable GitlabSecurity/PublicSend true rescue => e - Rails.logger.error("Unable to connect to Akismet: #{e}, skipping!") + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping!") # rubocop:disable Gitlab/RailsLogger false end end diff --git a/app/services/application_settings/update_service.rb b/app/services/application_settings/update_service.rb index 7eeaf8aade1..8115585b7a8 100644 --- a/app/services/application_settings/update_service.rb +++ b/app/services/application_settings/update_service.rb @@ -7,7 +7,7 @@ module ApplicationSettings attr_reader :params, :application_setting def execute - validate_classification_label(application_setting, :external_authorization_service_default_label) + validate_classification_label(application_setting, :external_authorization_service_default_label) unless bypass_external_auth? if application_setting.errors.any? return false @@ -15,6 +15,8 @@ module ApplicationSettings update_terms(@params.delete(:terms)) + add_to_outbound_local_requests_whitelist(@params.delete(:add_to_outbound_local_requests_whitelist)) + if params.key?(:performance_bar_allowed_group_path) params[:performance_bar_allowed_group_id] = performance_bar_allowed_group_id end @@ -32,6 +34,13 @@ module ApplicationSettings params.key?(:usage_ping_enabled) || params.key?(:version_check_enabled) end + def add_to_outbound_local_requests_whitelist(values) + values_array = Array(values).reject(&:empty?) + return if values_array.empty? + + @application_setting.add_to_outbound_local_requests_whitelist(values_array) + end + def update_terms(terms) return unless terms.present? @@ -50,5 +59,9 @@ module ApplicationSettings Group.find_by_full_path(group_full_path)&.id if group_full_path.present? end + + def bypass_external_auth? + params.key?(:external_authorization_service_enabled) && !Gitlab::Utils.to_boolean(params[:external_authorization_service_enabled]) + end end end diff --git a/app/services/audit_event_service.rb b/app/services/audit_event_service.rb index 201048aaba5..73f3408a240 100644 --- a/app/services/audit_event_service.rb +++ b/app/services/audit_event_service.rb @@ -35,8 +35,12 @@ class AuditEventService @file_logger ||= Gitlab::AuditJsonLogger.build end + def formatted_details + @details.merge(@details.slice(:from, :to).transform_values(&:to_s)) + end + def log_security_event_to_file - file_logger.info(base_payload.merge(@details)) + file_logger.info(base_payload.merge(formatted_details)) end def log_security_event_to_database diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 707caee482c..0a069320936 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -17,6 +17,14 @@ module Auth end def self.full_access_token(*names) + access_token(%w(*), names) + end + + def self.pull_access_token(*names) + access_token(['pull'], names) + end + + def self.access_token(actions, names) names = names.flatten registry = Gitlab.config.registry token = JSONWebToken::RSAToken.new(registry.key) @@ -25,7 +33,7 @@ module Auth token.expire_time = token_expire_at token[:access] = names.map do |name| - { type: 'repository', name: name, actions: %w(*) } + { type: 'repository', name: name, actions: actions } end token.encoded diff --git a/app/services/award_emojis/add_service.rb b/app/services/award_emojis/add_service.rb new file mode 100644 index 00000000000..eac15dabbf0 --- /dev/null +++ b/app/services/award_emojis/add_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module AwardEmojis + class AddService < AwardEmojis::BaseService + include Gitlab::Utils::StrongMemoize + + def execute + unless awardable.user_can_award?(current_user) + return error('User cannot award emoji to awardable', status: :forbidden) + end + + unless awardable.emoji_awardable? + return error('Awardable cannot be awarded emoji', status: :unprocessable_entity) + end + + award = awardable.award_emoji.create(name: name, user: current_user) + + if award.persisted? + TodoService.new.new_award_emoji(todoable, current_user) if todoable + success(award: award) + else + error(award.errors.full_messages, award: award) + end + end + + private + + def todoable + strong_memoize(:todoable) do + case awardable + when Note + # We don't create todos for personal snippet comments for now + awardable.noteable unless awardable.for_personal_snippet? + when MergeRequest, Issue + awardable + when Snippet + nil + end + end + end + end +end diff --git a/app/services/award_emojis/base_service.rb b/app/services/award_emojis/base_service.rb new file mode 100644 index 00000000000..a677d03a221 --- /dev/null +++ b/app/services/award_emojis/base_service.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module AwardEmojis + class BaseService < ::BaseService + attr_accessor :awardable, :name + + def initialize(awardable, name, current_user) + @awardable = awardable + @name = normalize_name(name) + + super(awardable.project, current_user) + end + + private + + def normalize_name(name) + Gitlab::Emoji.normalize_emoji_name(name) + end + + # Provide more error state data than what BaseService allows. + # - An array of errors + # - The `AwardEmoji` if present + def error(errors, award: nil, status: nil) + errors = Array.wrap(errors) + + super(errors.to_sentence.presence, status).merge({ + award: award, + errors: errors + }) + end + end +end diff --git a/app/services/award_emojis/collect_user_emoji_service.rb b/app/services/award_emojis/collect_user_emoji_service.rb new file mode 100644 index 00000000000..6cab23f3edf --- /dev/null +++ b/app/services/award_emojis/collect_user_emoji_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Class for retrieving information about emoji awarded _by_ a particular user. +module AwardEmojis + class CollectUserEmojiService + attr_reader :current_user + + # current_user - The User to generate the data for. + def initialize(current_user = nil) + @current_user = current_user + end + + def execute + return [] unless current_user + + # We want the resulting data set to be an Array containing the emoji names + # in descending order, based on how often they were awarded. + AwardEmoji + .award_counts_for_user(current_user) + .map { |name, _| { name: name } } + end + end +end diff --git a/app/services/award_emojis/destroy_service.rb b/app/services/award_emojis/destroy_service.rb new file mode 100644 index 00000000000..3789a8403bc --- /dev/null +++ b/app/services/award_emojis/destroy_service.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module AwardEmojis + class DestroyService < AwardEmojis::BaseService + def execute + unless awardable.user_can_award?(current_user) + return error('User cannot destroy emoji on the awardable', status: :forbidden) + end + + awards = AwardEmojisFinder.new(awardable, name: name, awarded_by: current_user).execute + + if awards.empty? + return error("User has not awarded emoji of type #{name} on the awardable", status: :forbidden) + end + + award = awards.destroy_all.first # rubocop: disable DestroyAll + + success(award: award) + end + end +end diff --git a/app/services/award_emojis/toggle_service.rb b/app/services/award_emojis/toggle_service.rb new file mode 100644 index 00000000000..812dd1c2889 --- /dev/null +++ b/app/services/award_emojis/toggle_service.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module AwardEmojis + class ToggleService < AwardEmojis::BaseService + def execute + if awardable.awarded_emoji?(name, current_user) + DestroyService.new(awardable, name, current_user).execute + else + AddService.new(awardable, name, current_user).execute + end + end + end +end diff --git a/app/services/boards/issues/move_service.rb b/app/services/boards/issues/move_service.rb index e27d34dbcab..00ce27db7c8 100644 --- a/app/services/boards/issues/move_service.rb +++ b/app/services/boards/issues/move_service.rb @@ -4,14 +4,62 @@ module Boards module Issues class MoveService < Boards::BaseService def execute(issue) - return false unless can?(current_user, :update_issue, issue) - return false if issue_params(issue).empty? + issue_modification_params = issue_params(issue) + return false if issue_modification_params.empty? - update(issue) + move_single_issue(issue, issue_modification_params) + end + + def execute_multiple(issues) + return execute_multiple_empty_result if issues.empty? + + handled_issues = [] + last_inserted_issue_id = nil + count = issues.each.inject(0) do |moved_count, issue| + issue_modification_params = issue_params(issue) + next moved_count if issue_modification_params.empty? + + if last_inserted_issue_id + issue_modification_params[:move_between_ids] = move_below(last_inserted_issue_id) + end + + last_inserted_issue_id = issue.id + handled_issue = move_single_issue(issue, issue_modification_params) + handled_issues << present_issue_entity(handled_issue) if handled_issue + handled_issue && handled_issue.valid? ? moved_count + 1 : moved_count + end + + { + count: count, + success: count == issues.size, + issues: handled_issues + } end private + def present_issue_entity(issue) + ::API::Entities::Issue.represent(issue) + end + + def execute_multiple_empty_result + @execute_multiple_empty_result ||= { + count: 0, + success: false, + issues: [] + } + end + + def move_below(id) + move_between_ids({ move_after_id: nil, move_before_id: id }) + end + + def move_single_issue(issue, issue_modification_params) + return unless can?(current_user, :update_issue, issue) + + update(issue, issue_modification_params) + end + def board @board ||= parent.boards.find(params[:board_id]) end @@ -33,8 +81,8 @@ module Boards end # rubocop: enable CodeReuse/ActiveRecord - def update(issue) - ::Issues::UpdateService.new(issue.project, current_user, issue_params(issue)).execute(issue) + def update(issue, issue_modification_params) + ::Issues::UpdateService.new(issue.project, current_user, issue_modification_params).execute(issue) end def issue_params(issue) @@ -48,6 +96,7 @@ module Boards ) end + move_between_ids = move_between_ids(params) if move_between_ids attrs[:move_between_ids] = move_between_ids attrs[:board_group_id] = board.group&.id @@ -78,8 +127,8 @@ module Boards end # rubocop: enable CodeReuse/ActiveRecord - def move_between_ids - ids = [params[:move_after_id], params[:move_before_id]] + def move_between_ids(move_params) + ids = [move_params[:move_after_id], move_params[:move_before_id]] .map(&:to_i) .map { |m| m.positive? ? m : nil } diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index a1dd00721b5..700d78361a4 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -2,8 +2,25 @@ module Ci class ArchiveTraceService - def execute(job) + def execute(job, worker_name:) + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_live_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have live trace but going to be archived.', + job_id: job.id) + return + end + job.trace.archive! + + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_archived_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have archived trace after archiving.', + job_id: job.id) + end rescue ::Gitlab::Ci::Trace::AlreadyArchivedError # It's already archived, thus we can safely ignore this exception. rescue => e @@ -11,7 +28,7 @@ module Ci # If `archive!` keeps failing for over a week, that could incur data loss. # (See more https://docs.gitlab.com/ee/administration/job_traces.html#new-live-trace-architecture) # In order to avoid interrupting the system, we do not raise an exception here. - archive_error(e, job) + archive_error(e, job, worker_name) end private @@ -22,13 +39,16 @@ module Ci "Counter of failed attempts of trace archiving") end - def archive_error(error, job) + def archive_error(error, job, worker_name) failed_archive_counter.increment - Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}" + + Sidekiq.logger.warn(class: worker_name, + message: "Failed to archive trace. message: #{error.message}.", + job_id: job.id) Gitlab::Sentry .track_exception(error, - issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502', + issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502', extra: { job_id: job.id }) end end diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb index eb0b070657d..9f922ffde81 100644 --- a/app/services/ci/play_build_service.rb +++ b/app/services/ci/play_build_service.rb @@ -2,7 +2,7 @@ module Ci class PlayBuildService < ::BaseService - def execute(build) + def execute(build, job_variables_attributes = nil) unless can?(current_user, :update_build, build) raise Gitlab::Access::AccessDeniedError end @@ -10,7 +10,7 @@ module Ci # Try to enqueue the build, otherwise create a duplicate. # if build.enqueue - build.tap { |action| action.update(user: current_user) } + build.tap { |action| action.update(user: current_user, job_variables_attributes: job_variables_attributes || []) } else Ci::Build.retry(build, current_user) end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 4a7ce00b8e2..3b145a65d79 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -4,56 +4,105 @@ module Ci class ProcessPipelineService < BaseService attr_reader :pipeline - def execute(pipeline) + def execute(pipeline, trigger_build_ids = nil) @pipeline = pipeline update_retried - new_builds = - stage_indexes_of_created_processables.map do |index| - process_stage(index) - end + success = process_stages_without_needs + + # we evaluate dependent needs, + # only when the another job has finished + success = process_builds_with_needs(trigger_build_ids) || success @pipeline.update_status - new_builds.flatten.any? + success end private - def process_stage(index) + def process_stages_without_needs + stage_indexes_of_created_processables_without_needs.flat_map do |index| + process_stage_without_needs(index) + end.any? + end + + def process_stage_without_needs(index) current_status = status_for_prior_stages(index) - return if HasStatus::BLOCKED_STATUS.include?(current_status) + return unless HasStatus::COMPLETED_STATUSES.include?(current_status) - if HasStatus::COMPLETED_STATUSES.include?(current_status) - created_processables_in_stage(index).select do |build| - Gitlab::OptimisticLocking.retry_lock(build) do |subject| - Ci::ProcessBuildService.new(project, @user) - .execute(build, current_status) - end - end + created_processables_in_stage_without_needs(index).select do |build| + process_build(build, current_status) + end + end + + def process_builds_with_needs(trigger_build_ids) + return false unless trigger_build_ids.present? + return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true) + + # we find processables that are dependent: + # 1. because of current dependency, + trigger_build_names = pipeline.processables.latest + .for_ids(trigger_build_ids).names + + # 2. does not have builds that not yet complete + incomplete_build_names = pipeline.processables.latest + .incomplete.names + + # Each found processable is guaranteed here to have completed status + created_processables + .with_needs(trigger_build_names) + .without_needs(incomplete_build_names) + .find_each + .map(&method(:process_build_with_needs)) + .any? + end + + def process_build_with_needs(build) + current_status = status_for_build_needs(build.needs.map(&:name)) + + return unless HasStatus::COMPLETED_STATUSES.include?(current_status) + + process_build(build, current_status) + end + + def process_build(build, current_status) + Gitlab::OptimisticLocking.retry_lock(build) do |subject| + Ci::ProcessBuildService.new(project, @user) + .execute(subject, current_status) end end - # rubocop: disable CodeReuse/ActiveRecord def status_for_prior_stages(index) - pipeline.builds.where('stage_idx < ?', index).latest.status || 'success' + pipeline.processables.status_for_prior_stages(index) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord - def stage_indexes_of_created_processables - created_processables.order(:stage_idx).pluck('distinct stage_idx') + def status_for_build_needs(needs) + pipeline.processables.status_for_names(needs) end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord - def created_processables_in_stage(index) - created_processables.where(stage_idx: index) + def stage_indexes_of_created_processables_without_needs + created_processables_without_needs.order(:stage_idx) + .pluck(Arel.sql('DISTINCT stage_idx')) end # rubocop: enable CodeReuse/ActiveRecord + def created_processables_in_stage_without_needs(index) + created_processables_without_needs + .for_stage(index) + end + + def created_processables_without_needs + if Feature.enabled?(:ci_dag_support, project, default_enabled: true) + pipeline.processables.created.without_needs + else + pipeline.processables.created + end + end + def created_processables pipeline.processables.created end @@ -68,7 +117,7 @@ module Ci latest_statuses = pipeline.statuses.latest .group(:name) .having('count(*) > 1') - .pluck('max(id)', 'name') + .pluck(Arel.sql('MAX(id)'), 'name') # mark builds that are retried pipeline.statuses.latest diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index fab8a179843..338495ba030 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -5,7 +5,7 @@ module Ci CLONE_ACCESSORS = %i[pipeline project ref tag options name allow_failure stage stage_id stage_idx trigger_request yaml_variables when environment coverage_regex - description tag_list protected].freeze + description tag_list protected needs].freeze def execute(build) reprocess!(build).tap do |new_build| diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb index 9c589d910eb..31c7178c9e7 100644 --- a/app/services/ci/update_build_queue_service.rb +++ b/app/services/ci/update_build_queue_service.rb @@ -9,6 +9,10 @@ module Ci private def tick_for(build, runners) + if Feature.enabled?(:ci_update_queues_for_online_runners, build.project, default_enabled: true) + runners = runners.with_recent_runner_queue + end + runners.each do |runner| runner.pick_build!(build) end diff --git a/app/services/clusters/applications/check_uninstall_progress_service.rb b/app/services/clusters/applications/check_uninstall_progress_service.rb index 8786d295d6a..e51d84ef052 100644 --- a/app/services/clusters/applications/check_uninstall_progress_service.rb +++ b/app/services/clusters/applications/check_uninstall_progress_service.rb @@ -23,6 +23,7 @@ module Clusters private def on_success + app.post_uninstall app.destroy! rescue StandardError => e app.make_errored!(_('Application uninstalled but failed to destroy: %{error_message}') % { error_message: e.message }) diff --git a/app/services/clusters/build_kubernetes_namespace_service.rb b/app/services/clusters/build_kubernetes_namespace_service.rb new file mode 100644 index 00000000000..2574f77bbf9 --- /dev/null +++ b/app/services/clusters/build_kubernetes_namespace_service.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Clusters + class BuildKubernetesNamespaceService + attr_reader :cluster, :environment + + def initialize(cluster, environment:) + @cluster = cluster + @environment = environment + end + + def execute + cluster.kubernetes_namespaces.build(attributes) + end + + private + + def attributes + attributes = { + project: environment.project, + namespace: namespace, + service_account_name: "#{namespace}-service-account" + } + + attributes[:cluster_project] = cluster.cluster_project if cluster.project_type? + attributes[:environment] = environment if cluster.namespace_per_environment? + + attributes + end + + def namespace + Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: environment.project).from_environment_slug(environment.slug) + end + end +end diff --git a/app/services/clusters/create_service.rb b/app/services/clusters/create_service.rb index 886e484caaf..e5a5b73321a 100644 --- a/app/services/clusters/create_service.rb +++ b/app/services/clusters/create_service.rb @@ -10,28 +10,36 @@ module Clusters def execute(access_token: nil) raise ArgumentError, 'Unknown clusterable provided' unless clusterable - raise ArgumentError, _('Instance does not support multiple Kubernetes clusters') unless can_create_cluster? - cluster_params = params.merge(user: current_user).merge(clusterable_params) + cluster_params = params.merge(global_params).merge(clusterable_params) + cluster_params[:provider_gcp_attributes].try do |provider| provider[:access_token] = access_token end - create_cluster(cluster_params).tap do |cluster| - ClusterProvisionWorker.perform_async(cluster.id) if cluster.persisted? + cluster = Clusters::Cluster.new(cluster_params) + + unless can_create_cluster? + cluster.errors.add(:base, _('Instance does not support multiple Kubernetes clusters')) end - end - private + return cluster if cluster.errors.present? - def create_cluster(cluster_params) - Clusters::Cluster.create(cluster_params) + cluster.tap do |cluster| + cluster.save && ClusterProvisionWorker.perform_async(cluster.id) + end end + private + def clusterable @clusterable ||= params.delete(:clusterable) end + def global_params + { user: current_user, namespace_per_environment: Feature.enabled?(:kubernetes_namespace_per_environment, default_enabled: true) } + end + def clusterable_params case clusterable when ::Project diff --git a/app/services/clusters/gcp/kubernetes.rb b/app/services/clusters/gcp/kubernetes.rb index 90ed529670c..85711764785 100644 --- a/app/services/clusters/gcp/kubernetes.rb +++ b/app/services/clusters/gcp/kubernetes.rb @@ -9,6 +9,8 @@ module Clusters GITLAB_CLUSTER_ROLE_BINDING_NAME = 'gitlab-admin' GITLAB_CLUSTER_ROLE_NAME = 'cluster-admin' PROJECT_CLUSTER_ROLE_NAME = 'edit' + GITLAB_KNATIVE_SERVING_ROLE_NAME = 'gitlab-knative-serving-role' + GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME = 'gitlab-knative-serving-rolebinding' end end end diff --git a/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb b/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb index 806f320381d..c45dac7b273 100644 --- a/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb +++ b/app/services/clusters/gcp/kubernetes/create_or_update_namespace_service.rb @@ -11,7 +11,6 @@ module Clusters end def execute - configure_kubernetes_namespace create_project_service_account configure_kubernetes_token @@ -22,10 +21,6 @@ module Clusters attr_reader :cluster, :kubernetes_namespace, :platform - def configure_kubernetes_namespace - kubernetes_namespace.set_defaults - end - def create_project_service_account Clusters::Gcp::Kubernetes::CreateOrUpdateServiceAccountService.namespace_creator( platform.kubeclient, diff --git a/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb b/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb index 49e766cbf13..7c5450dbcd6 100644 --- a/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb +++ b/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb @@ -41,7 +41,15 @@ module Clusters kubeclient.create_or_update_service_account(service_account_resource) kubeclient.create_or_update_secret(service_account_token_resource) - create_role_or_cluster_role_binding if rbac + + return unless rbac + + create_role_or_cluster_role_binding + + return unless namespace_creator + + create_or_update_knative_serving_role + create_or_update_knative_serving_role_binding end private @@ -63,6 +71,14 @@ module Clusters end end + def create_or_update_knative_serving_role + kubeclient.update_role(knative_serving_role_resource) + end + + def create_or_update_knative_serving_role_binding + kubeclient.update_role_binding(knative_serving_role_binding_resource) + end + def service_account_resource Gitlab::Kubernetes::ServiceAccount.new( service_account_name, @@ -92,6 +108,29 @@ module Clusters Gitlab::Kubernetes::RoleBinding.new( name: role_binding_name, role_name: Clusters::Gcp::Kubernetes::PROJECT_CLUSTER_ROLE_NAME, + role_kind: :ClusterRole, + namespace: service_account_namespace, + service_account_name: service_account_name + ).generate + end + + def knative_serving_role_resource + Gitlab::Kubernetes::Role.new( + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + namespace: service_account_namespace, + rules: [{ + apiGroups: %w(serving.knative.dev), + resources: %w(configurations configurationgenerations routes revisions revisionuids autoscalers services), + verbs: %w(get list create update delete patch watch) + }] + ).generate + end + + def knative_serving_role_binding_resource + Gitlab::Kubernetes::RoleBinding.new( + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME, + role_name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + role_kind: :Role, namespace: service_account_namespace, service_account_name: service_account_name ).generate diff --git a/app/services/clusters/refresh_service.rb b/app/services/clusters/refresh_service.rb deleted file mode 100644 index 3752a306793..00000000000 --- a/app/services/clusters/refresh_service.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Clusters - class RefreshService - def self.create_or_update_namespaces_for_cluster(cluster) - projects_with_missing_kubernetes_namespaces_for_cluster(cluster).each do |project| - create_or_update_namespace(cluster, project) - end - end - - def self.create_or_update_namespaces_for_project(project) - clusters_with_missing_kubernetes_namespaces_for_project(project).each do |cluster| - create_or_update_namespace(cluster, project) - end - end - - def self.projects_with_missing_kubernetes_namespaces_for_cluster(cluster) - cluster.all_projects.missing_kubernetes_namespace(cluster.kubernetes_namespaces) - end - - private_class_method :projects_with_missing_kubernetes_namespaces_for_cluster - - def self.clusters_with_missing_kubernetes_namespaces_for_project(project) - project.clusters.managed.missing_kubernetes_namespace(project.kubernetes_namespaces) - end - - private_class_method :clusters_with_missing_kubernetes_namespaces_for_project - - def self.create_or_update_namespace(cluster, project) - kubernetes_namespace = cluster.find_or_initialize_kubernetes_namespace_for_project(project) - - ::Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService.new( - cluster: cluster, - kubernetes_namespace: kubernetes_namespace - ).execute - end - - private_class_method :create_or_update_namespace - end -end diff --git a/app/services/cohorts_service.rb b/app/services/cohorts_service.rb index 6d466c2fc9c..97fbb70f350 100644 --- a/app/services/cohorts_service.rb +++ b/app/services/cohorts_service.rb @@ -95,10 +95,6 @@ class CohortsService # rubocop: enable CodeReuse/ActiveRecord def column_to_date(column) - if Gitlab::Database.postgresql? - "CAST(DATE_TRUNC('month', #{column}) AS date)" - else - "STR_TO_DATE(DATE_FORMAT(#{column}, '%Y-%m-01'), '%Y-%m-%d')" - end + "CAST(DATE_TRUNC('month', #{column}) AS date)" end end diff --git a/app/services/commits/create_service.rb b/app/services/commits/create_service.rb index bb34a3d3352..f3be68f9602 100644 --- a/app/services/commits/create_service.rb +++ b/app/services/commits/create_service.rb @@ -10,6 +10,7 @@ module Commits @start_project = params[:start_project] || @project @start_branch = params[:start_branch] + @start_sha = params[:start_sha] @branch_name = params[:branch_name] @force = params[:force] || false end @@ -40,7 +41,7 @@ module Commits end def different_branch? - @start_branch != @branch_name || @start_project != @project + @start_project != @project || @start_branch != @branch_name || @start_sha.present? end def force? @@ -49,6 +50,7 @@ module Commits def validate! validate_permissions! + validate_start_sha! validate_on_branch! validate_branch_existence! @@ -63,7 +65,21 @@ module Commits end end + def validate_start_sha! + return unless @start_sha + + if @start_branch + raise_error("You can't pass both start_branch and start_sha") + elsif !Gitlab::Git.commit_id?(@start_sha) + raise_error("Invalid start_sha '#{@start_sha}'") + elsif !@start_project.repository.commit(@start_sha) + raise_error("Cannot find start_sha '#{@start_sha}'") + end + end + def validate_on_branch! + return unless @start_branch + if !@start_project.empty_repo? && !@start_project.repository.branch_exists?(@start_branch) raise_error('You can only create or edit files when you are on a branch') end diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb index 2cb73555d85..0c5ecca3a50 100644 --- a/app/services/concerns/exclusive_lease_guard.rb +++ b/app/services/concerns/exclusive_lease_guard.rb @@ -58,6 +58,6 @@ module ExclusiveLeaseGuard end def log_error(message, extra_args = {}) - Rails.logger.error(message) + Rails.logger.error(message) # rubocop:disable Gitlab/RailsLogger end end diff --git a/app/services/create_snippet_service.rb b/app/services/create_snippet_service.rb index 6f1fce4989e..6e5bf823cc7 100644 --- a/app/services/create_snippet_service.rb +++ b/app/services/create_snippet_service.rb @@ -23,6 +23,7 @@ class CreateSnippetService < BaseService if snippet.save UserAgentDetailService.new(snippet, @request).create + Gitlab::UsageDataCounters::SnippetCounter.count(:create) end snippet diff --git a/app/services/files/multi_service.rb b/app/services/files/multi_service.rb index d8c4e5bc5e8..65af4dd5a28 100644 --- a/app/services/files/multi_service.rb +++ b/app/services/files/multi_service.rb @@ -47,6 +47,7 @@ module Files author_name: @author_name, start_project: @start_project, start_branch_name: @start_branch, + start_sha: @start_sha, force: force? ) rescue ArgumentError => e diff --git a/app/services/git/base_hooks_service.rb b/app/services/git/base_hooks_service.rb index d30df34e54b..47c308c8280 100644 --- a/app/services/git/base_hooks_service.rb +++ b/app/services/git/base_hooks_service.rb @@ -8,8 +8,6 @@ module Git PROCESS_COMMIT_LIMIT = 100 def execute - project.repository.after_create if project.empty_repo? - create_events create_pipelines execute_project_hooks @@ -19,7 +17,7 @@ module Git update_remote_mirrors - push_data + success end private @@ -33,7 +31,7 @@ module Git end def limited_commits - commits.last(PROCESS_COMMIT_LIMIT) + @limited_commits ||= commits.last(PROCESS_COMMIT_LIMIT) end def commits_count @@ -48,43 +46,69 @@ module Git [] end + # Push events in the activity feed only show information for the + # last commit. def create_events - EventCreateService.new.push(project, current_user, push_data) + EventCreateService.new.push(project, current_user, event_push_data) end def create_pipelines return unless params.fetch(:create_pipelines, true) Ci::CreatePipelineService - .new(project, current_user, push_data) + .new(project, current_user, pipeline_params) .execute(:push, pipeline_options) end def execute_project_hooks - project.execute_hooks(push_data, hook_name) - project.execute_services(push_data, hook_name) + # Creating push_data invokes one CommitDelta RPC per commit. Only + # build this data if we actually need it. + project.execute_hooks(push_data, hook_name) if project.has_active_hooks?(hook_name) + project.execute_services(push_data, hook_name) if project.has_active_services?(hook_name) end def enqueue_invalidate_cache - ProjectCacheWorker.perform_async( - project.id, - invalidated_file_types, - [:commit_count, :repository_size] - ) + file_types = invalidated_file_types + + return unless file_types.present? + + ProjectCacheWorker.perform_async(project.id, file_types, [], false) end - def push_data - @push_data ||= Gitlab::DataBuilder::Push.build( - project: project, - user: current_user, + def pipeline_params + { + before: params[:oldrev], + after: params[:newrev], + ref: params[:ref], + push_options: params[:push_options] || {}, + checkout_sha: Gitlab::DataBuilder::Push.checkout_sha( + project.repository, params[:newrev], params[:ref]) + } + end + + def push_data_params(commits:, with_changed_files: true) + { oldrev: params[:oldrev], newrev: params[:newrev], ref: params[:ref], - commits: limited_commits, + project: project, + user: current_user, + commits: commits, message: event_message, commits_count: commits_count, - push_options: params[:push_options] || {} - ) + with_changed_files: with_changed_files + } + end + + def event_push_data + # We only need the last commit for the event push, and we don't + # need the full deltas either. + @event_push_data ||= Gitlab::DataBuilder::Push.build( + push_data_params(commits: commits.last, with_changed_files: false)) + end + + def push_data + @push_data ||= Gitlab::DataBuilder::Push.build(push_data_params(commits: limited_commits)) # Dependent code may modify the push data, so return a duplicate each time @push_data.dup diff --git a/app/services/git/branch_hooks_service.rb b/app/services/git/branch_hooks_service.rb index c41f445c3c4..d2b037a680c 100644 --- a/app/services/git/branch_hooks_service.rb +++ b/app/services/git/branch_hooks_service.rb @@ -63,7 +63,7 @@ module Git end def branch_create_hooks - project.repository.after_create_branch + project.repository.after_create_branch(expire_cache: false) project.after_create_default_branch if default_branch? end @@ -78,13 +78,21 @@ module Git end def branch_remove_hooks - project.repository.after_remove_branch + project.repository.after_remove_branch(expire_cache: false) end # Schedules processing of commit messages def enqueue_process_commit_messages - limited_commits.each do |commit| - next unless commit.matches_cross_reference_regex? + referencing_commits = limited_commits.select(&:matches_cross_reference_regex?) + + upstream_commit_ids = upstream_commit_ids(referencing_commits) + + referencing_commits.each do |commit| + # Avoid reprocessing commits that already exist upstream if the project + # is a fork. This will prevent duplicated/superfluous system notes on + # mentionables referenced by a commit that is pushed to the upstream, + # that is then also pushed to forks when these get synced by users. + next if upstream_commit_ids.include?(commit.id) ProcessCommitWorker.perform_async( project.id, @@ -142,5 +150,18 @@ module Git def branch_name strong_memoize(:branch_name) { Gitlab::Git.ref_name(params[:ref]) } end + + def upstream_commit_ids(commits) + set = Set.new + + upstream_project = project.fork_source + if upstream_project + upstream_project + .commits_by(oids: commits.map(&:id)) + .each { |commit| set << commit.id } + end + + set + end end end diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb index e9659f5489a..e78e5d5fc2c 100644 --- a/app/services/groups/create_service.rb +++ b/app/services/groups/create_service.rb @@ -27,8 +27,7 @@ module Groups @group.build_chat_team(name: response['name'], team_id: response['id']) end - @group.save - @group.add_owner(current_user) + @group.add_owner(current_user) if @group.save @group end diff --git a/app/services/groups/destroy_service.rb b/app/services/groups/destroy_service.rb index 654fe84e3dc..9e00cbbbc55 100644 --- a/app/services/groups/destroy_service.rb +++ b/app/services/groups/destroy_service.rb @@ -6,7 +6,7 @@ module Groups def async_execute job_id = GroupDestroyWorker.perform_async(group.id, current_user.id) - Rails.logger.info("User #{current_user.id} scheduled a deletion of group ID #{group.id} with job ID #{job_id}") + Rails.logger.info("User #{current_user.id} scheduled a deletion of group ID #{group.id} with job ID #{job_id}") # rubocop:disable Gitlab/RailsLogger end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/services/groups/nested_create_service.rb b/app/services/groups/nested_create_service.rb index 01bd685712b..a51ac9aa593 100644 --- a/app/services/groups/nested_create_service.rb +++ b/app/services/groups/nested_create_service.rb @@ -18,10 +18,6 @@ module Groups return namespace end - if group_path.include?('/') && !Group.supports_nested_objects? - raise 'Nested groups are not supported on MySQL' - end - create_group_path end diff --git a/app/services/groups/transfer_service.rb b/app/services/groups/transfer_service.rb index 98e7c311572..fe7e07ef9f0 100644 --- a/app/services/groups/transfer_service.rb +++ b/app/services/groups/transfer_service.rb @@ -43,7 +43,6 @@ module Groups def ensure_allowed_transfer raise_transfer_error(:group_is_already_root) if group_is_already_root? - raise_transfer_error(:database_not_supported) unless Group.supports_nested_objects? raise_transfer_error(:same_parent_as_current) if same_parent? raise_transfer_error(:invalid_policies) unless valid_policies? raise_transfer_error(:namespace_with_same_path) if namespace_with_same_path? diff --git a/app/services/groups/update_service.rb b/app/services/groups/update_service.rb index 73e1e00dc33..116756bacfe 100644 --- a/app/services/groups/update_service.rb +++ b/app/services/groups/update_service.rb @@ -46,6 +46,11 @@ module Groups params.delete(:parent_id) end + # overridden in EE + def remove_unallowed_params + params.delete(:emails_disabled) unless can?(current_user, :set_emails_disabled, group) + end + def valid_share_with_group_lock_change? return true unless changing_share_with_group_lock? return true if can?(current_user, :change_share_with_group_lock, group) diff --git a/app/services/issuable/bulk_update_service.rb b/app/services/issuable/bulk_update_service.rb index 6d215d7a3b9..273a12f386a 100644 --- a/app/services/issuable/bulk_update_service.rb +++ b/app/services/issuable/bulk_update_service.rb @@ -29,7 +29,7 @@ module Issuable items.each do |issuable| next unless can?(current_user, :"update_#{type}", issuable) - update_class.new(issuable.project, current_user, params).execute(issuable) + update_class.new(issuable.issuing_parent, current_user, params).execute(issuable) end { diff --git a/app/services/issuable/clone/attributes_rewriter.rb b/app/services/issuable/clone/attributes_rewriter.rb index 0300cc0d8d3..3c061d35558 100644 --- a/app/services/issuable/clone/attributes_rewriter.rb +++ b/app/services/issuable/clone/attributes_rewriter.rb @@ -17,6 +17,8 @@ module Issuable private def cloneable_milestone + return unless new_entity.supports_milestone? + title = original_entity.milestone&.title return unless title diff --git a/app/services/issuable/clone/content_rewriter.rb b/app/services/issuable/clone/content_rewriter.rb index 00d7078859d..f75b51c4be3 100644 --- a/app/services/issuable/clone/content_rewriter.rb +++ b/app/services/issuable/clone/content_rewriter.rb @@ -23,10 +23,14 @@ module Issuable end def rewrite_notes + new_discussion_ids = {} original_entity.notes_with_associations.find_each do |note| new_note = note.dup + new_discussion_ids[note.discussion_id] ||= Discussion.discussion_id(new_note) new_params = { - project: new_entity.project, noteable: new_entity, + project: new_entity.project, + noteable: new_entity, + discussion_id: new_discussion_ids[note.discussion_id], note: rewrite_content(new_note.note), note_html: nil, created_at: note.created_at, diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index db673cace81..2ab6e88599f 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -344,10 +344,7 @@ class IssuableBaseService < BaseService def toggle_award(issuable) award = params.delete(:emoji_award) - if award - todo_service.new_award_emoji(issuable, current_user) - issuable.toggle_award_emoji(award, current_user) - end + AwardEmojis::ToggleService.new(issuable, award, current_user).execute if award end def associations_before_update(issuable) @@ -358,6 +355,7 @@ class IssuableBaseService < BaseService assignees: issuable.assignees.to_a } associations[:total_time_spent] = issuable.total_time_spent if issuable.respond_to?(:total_time_spent) + associations[:description] = issuable.description associations end diff --git a/app/services/issues/update_service.rb b/app/services/issues/update_service.rb index 7cd825aa967..c8f4412c9f2 100644 --- a/app/services/issues/update_service.rb +++ b/app/services/issues/update_service.rb @@ -61,6 +61,8 @@ module Issues if added_mentions.present? notification_service.async.new_mentions_in_issue(issue, added_mentions, current_user) end + + ZoomNotesService.new(issue, project, current_user, old_description: old_associations[:description]).execute end def handle_task_changes(issuable) diff --git a/app/services/labels/create_service.rb b/app/services/labels/create_service.rb index db710bac900..c032985be42 100644 --- a/app/services/labels/create_service.rb +++ b/app/services/labels/create_service.rb @@ -20,7 +20,7 @@ module Labels label.save label else - Rails.logger.warn("target_params should contain :project or :group or :template, actual value: #{target_params}") + Rails.logger.warn("target_params should contain :project or :group or :template, actual value: #{target_params}") # rubocop:disable Gitlab/RailsLogger end end end diff --git a/app/services/members/base_service.rb b/app/services/members/base_service.rb index e78affff797..5d69418fb7d 100644 --- a/app/services/members/base_service.rb +++ b/app/services/members/base_service.rb @@ -51,7 +51,9 @@ module Members def enqueue_delete_todos(member) type = member.is_a?(GroupMember) ? 'Group' : 'Project' # don't enqueue immediately to prevent todos removal in case of a mistake - TodosDestroyer::EntityLeaveWorker.perform_in(Todo::WAIT_FOR_DELETE, member.user_id, member.source_id, type) + member.run_after_commit_or_now do + TodosDestroyer::EntityLeaveWorker.perform_in(Todo::WAIT_FOR_DELETE, member.user_id, member.source_id, type) + end end end end diff --git a/app/services/members/destroy_service.rb b/app/services/members/destroy_service.rb index c8d5e563cd8..0164760920f 100644 --- a/app/services/members/destroy_service.rb +++ b/app/services/members/destroy_service.rb @@ -31,7 +31,7 @@ module Members return unless member.is_a?(GroupMember) && member.user && member.group delete_project_members(member) - delete_subgroup_members(member) if Group.supports_nested_objects? + delete_subgroup_members(member) end def delete_project_members(member) diff --git a/app/services/merge_requests/base_service.rb b/app/services/merge_requests/base_service.rb index 067510a8a0a..c6aae4c28f2 100644 --- a/app/services/merge_requests/base_service.rb +++ b/app/services/merge_requests/base_service.rb @@ -17,11 +17,9 @@ module MergeRequests end def execute_hooks(merge_request, action = 'open', old_rev: nil, old_associations: {}) - if merge_request.project - merge_data = hook_data(merge_request, action, old_rev: old_rev, old_associations: old_associations) - merge_request.project.execute_hooks(merge_data, :merge_request_hooks) - merge_request.project.execute_services(merge_data, :merge_request_hooks) - end + merge_data = hook_data(merge_request, action, old_rev: old_rev, old_associations: old_associations) + merge_request.project.execute_hooks(merge_data, :merge_request_hooks) + merge_request.project.execute_services(merge_data, :merge_request_hooks) end def cleanup_environments(merge_request) diff --git a/app/services/merge_requests/build_service.rb b/app/services/merge_requests/build_service.rb index 109c964e577..b28f80939ae 100644 --- a/app/services/merge_requests/build_service.rb +++ b/app/services/merge_requests/build_service.rb @@ -11,15 +11,18 @@ module MergeRequests # https://gitlab.com/gitlab-org/gitlab-ce/issues/53658 merge_quick_actions_into_params!(merge_request, only: [:target_branch]) merge_request.merge_params['force_remove_source_branch'] = params.delete(:force_remove_source_branch) if params.has_key?(:force_remove_source_branch) - merge_request.assign_attributes(params) + # Assign the projects first so we can use policies for `filter_params` merge_request.author = current_user + merge_request.source_project = find_source_project + merge_request.target_project = find_target_project + + filter_params(merge_request) + merge_request.assign_attributes(params.to_h.compact) + merge_request.compare_commits = [] - merge_request.source_project = find_source_project - merge_request.target_project = find_target_project - merge_request.target_branch = find_target_branch - merge_request.can_be_created = projects_and_branches_valid? - ensure_milestone_available(merge_request) + merge_request.target_branch = find_target_branch + merge_request.can_be_created = projects_and_branches_valid? # compare branches only if branches are valid, otherwise # compare_branches may raise an error @@ -50,12 +53,14 @@ module MergeRequests to: :merge_request def find_source_project + source_project = project_from_params(:source_project) return source_project if source_project.present? && can?(current_user, :create_merge_request_from, source_project) project end def find_target_project + target_project = project_from_params(:target_project) return target_project if target_project.present? && can?(current_user, :create_merge_request_in, target_project) target_project = project.default_merge_request_target @@ -65,6 +70,17 @@ module MergeRequests project end + def project_from_params(param_name) + project_from_params = params.delete(param_name) + + id_param_name = :"#{param_name}_id" + if project_from_params.nil? && params[id_param_name] + project_from_params = Project.find_by_id(params.delete(id_param_name)) + end + + project_from_params + end + def find_target_branch target_branch || target_project.default_branch end diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 06e46595b95..a69678a4422 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -27,6 +27,7 @@ module MergeRequests issuable.cache_merge_request_closes_issues!(current_user) create_pipeline_for(issuable, current_user) issuable.update_head_pipeline + Gitlab::UsageDataCounters::MergeRequestCounter.count(:create) super end diff --git a/app/services/merge_requests/merge_service.rb b/app/services/merge_requests/merge_service.rb index 3e0f5aa181c..6309052244d 100644 --- a/app/services/merge_requests/merge_service.rb +++ b/app/services/merge_requests/merge_service.rb @@ -113,12 +113,12 @@ module MergeRequests end def handle_merge_error(log_message:, save_message_on_model: false) - Rails.logger.error("MergeService ERROR: #{merge_request_info} - #{log_message}") + Rails.logger.error("MergeService ERROR: #{merge_request_info} - #{log_message}") # rubocop:disable Gitlab/RailsLogger @merge_request.update(merge_error: log_message) if save_message_on_model end def log_info(message) - @logger ||= Rails.logger + @logger ||= Rails.logger # rubocop:disable Gitlab/RailsLogger @logger.info("#{merge_request_info} - #{message}") end diff --git a/app/services/merge_requests/mergeability_check_service.rb b/app/services/merge_requests/mergeability_check_service.rb index 9fa50c9448f..962e2327b3e 100644 --- a/app/services/merge_requests/mergeability_check_service.rb +++ b/app/services/merge_requests/mergeability_check_service.rb @@ -3,6 +3,7 @@ module MergeRequests class MergeabilityCheckService < ::BaseService include Gitlab::Utils::StrongMemoize + include Gitlab::ExclusiveLeaseHelpers delegate :project, to: :@merge_request delegate :repository, to: :project @@ -21,13 +22,35 @@ module MergeRequests # where we need the current state of the merge ref in repository, the `recheck` # argument is required. # + # retry_lease - Concurrent calls wait for at least 10 seconds until the + # lease is granted (other process finishes running). Returns an error + # ServiceResponse if the lease is not granted during this time. + # # Returns a ServiceResponse indicating merge_status is/became can_be_merged # and the merge-ref is synced. Success in case of being/becoming mergeable, # error otherwise. - def execute(recheck: false) + def execute(recheck: false, retry_lease: true) return ServiceResponse.error(message: 'Invalid argument') unless merge_request return ServiceResponse.error(message: 'Unsupported operation') if Gitlab::Database.read_only? + return check_mergeability(recheck) unless merge_ref_auto_sync_lock_enabled? + + in_write_lock(retry_lease: retry_lease) do |retried| + # When multiple calls are waiting for the same lock (retry_lease), + # it's possible that when granted, the MR status was already updated for + # that object, therefore we reset if there was a lease retry. + merge_request.reset if retried + + check_mergeability(recheck) + end + rescue FailedToObtainLockError => error + ServiceResponse.error(message: error.message) + end + + private + + attr_reader :merge_request + def check_mergeability(recheck) recheck! if recheck update_merge_status @@ -46,9 +69,21 @@ module MergeRequests ServiceResponse.success(payload: payload) end - private + # It's possible for this service to send concurrent requests to Gitaly in order + # to "git update-ref" the same ref. Therefore we handle a light exclusive + # lease here. + # + def in_write_lock(retry_lease:, &block) + lease_key = "mergeability_check:#{merge_request.id}" - attr_reader :merge_request + lease_opts = { + ttl: 1.minute, + retries: retry_lease ? 10 : 0, + sleep_sec: retry_lease ? 1.second : 0 + } + + in_lock(lease_key, lease_opts, &block) + end def payload strong_memoize(:payload) do @@ -116,5 +151,9 @@ module MergeRequests def merge_ref_auto_sync_enabled? Feature.enabled?(:merge_ref_auto_sync, project, default_enabled: true) end + + def merge_ref_auto_sync_lock_enabled? + Feature.enabled?(:merge_ref_auto_sync_lock, project, default_enabled: true) + end end end diff --git a/app/services/merge_requests/push_options_handler_service.rb b/app/services/merge_requests/push_options_handler_service.rb index a24163331e8..b210004e6e1 100644 --- a/app/services/merge_requests/push_options_handler_service.rb +++ b/app/services/merge_requests/push_options_handler_service.rb @@ -117,15 +117,16 @@ module MergeRequests collect_errors_from_merge_request(merge_request) unless merge_request.valid? end - def create_params(branch) + def base_params params = { - assignees: [current_user], - source_branch: branch, - source_project: project, - target_branch: push_options[:target] || target_project.default_branch, - target_project: target_project + title: push_options[:title], + description: push_options[:description], + target_branch: push_options[:target], + force_remove_source_branch: push_options[:remove_source_branch] } + params.compact! + if push_options.key?(:merge_when_pipeline_succeeds) params.merge!( merge_when_pipeline_succeeds: push_options[:merge_when_pipeline_succeeds], @@ -136,23 +137,25 @@ module MergeRequests params end - def update_params - params = {} + def create_params(branch) + params = base_params - if push_options.key?(:merge_when_pipeline_succeeds) - params.merge!( - merge_when_pipeline_succeeds: push_options[:merge_when_pipeline_succeeds], - merge_user: current_user - ) - end + params.merge!( + assignees: [current_user], + source_branch: branch, + source_project: project, + target_project: target_project + ) - if push_options.key?(:target) - params[:target_branch] = push_options[:target] - end + params[:target_branch] ||= target_project.default_branch params end + def update_params + base_params + end + def collect_errors_from_merge_request(merge_request) merge_request.errors.full_messages.each do |error| errors << error diff --git a/app/services/merge_requests/rebase_service.rb b/app/services/merge_requests/rebase_service.rb index 8d3b9b05819..27c16ba1777 100644 --- a/app/services/merge_requests/rebase_service.rb +++ b/app/services/merge_requests/rebase_service.rb @@ -15,7 +15,8 @@ module MergeRequests end def rebase - if merge_request.gitaly_rebase_in_progress? + # Ensure Gitaly isn't already running a rebase + if source_project.repository.rebase_in_progress?(merge_request.id) log_error('Rebase task canceled: Another rebase is already in progress', save_message_on_model: true) return false end diff --git a/app/services/metrics/dashboard/base_embed_service.rb b/app/services/metrics/dashboard/base_embed_service.rb new file mode 100644 index 00000000000..8bb5f4892cb --- /dev/null +++ b/app/services/metrics/dashboard/base_embed_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Base class for embed services. Contains a few basic helper +# methods that the embed services share. +module Metrics + module Dashboard + class BaseEmbedService < ::Metrics::Dashboard::BaseService + def cache_key + "dynamic_metrics_dashboard_#{identifiers}" + end + + protected + + def dashboard_path + params[:dashboard_path].presence || + ::Metrics::Dashboard::SystemDashboardService::SYSTEM_DASHBOARD_PATH + end + + def group + params[:group] + end + + def title + params[:title] + end + + def y_label + params[:y_label] + end + + def identifiers + [dashboard_path, group, title, y_label].join('|') + end + end + end +end diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb new file mode 100644 index 00000000000..8a42675c66d --- /dev/null +++ b/app/services/metrics/dashboard/base_service.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +module Metrics + module Dashboard + class BaseService < ::BaseService + include Gitlab::Metrics::Dashboard::Errors + + def get_dashboard + return error('Insufficient permissions.', :unauthorized) unless allowed? + + success(dashboard: process_dashboard) + rescue StandardError => e + handle_errors(e) + end + + # Summary of all known dashboards for the service. + # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + def self.all_dashboard_paths(_project) + raise NotImplementedError + end + + # Returns an un-processed dashboard from the cache. + def raw_dashboard + Gitlab::Metrics::Dashboard::Cache.fetch(cache_key) { get_raw_dashboard } + end + + private + + # Determines whether users should be able to view + # dashboards at all. + def allowed? + Ability.allowed?(current_user, :read_environment, project) + end + + # Returns a new dashboard Hash, supplemented with DB info + def process_dashboard + Gitlab::Metrics::Dashboard::Processor + .new(project, params[:environment], raw_dashboard) + .process(insert_project_metrics: insert_project_metrics?) + end + + # @return [String] Relative filepath of the dashboard yml + def dashboard_path + params[:dashboard_path] + end + + # @return [Hash] an unmodified dashboard + def get_raw_dashboard + raise NotImplementedError + end + + # @return [String] + def cache_key + raise NotImplementedError + end + + # Determines whether custom metrics should be included + # in the processed output. + # @return [Boolean] + def insert_project_metrics? + false + end + end + end +end diff --git a/app/services/metrics/dashboard/custom_metric_embed_service.rb b/app/services/metrics/dashboard/custom_metric_embed_service.rb new file mode 100644 index 00000000000..50f070989fc --- /dev/null +++ b/app/services/metrics/dashboard/custom_metric_embed_service.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +# Responsible for returning a dashboard containing specified +# custom metrics. Creates panels based on the matching metrics +# stored in the database. +# +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class CustomMetricEmbedService < ::Metrics::Dashboard::BaseEmbedService + extend ::Gitlab::Utils::Override + include Gitlab::Utils::StrongMemoize + include Gitlab::Metrics::Dashboard::Defaults + + class << self + # Determines whether the provided params are sufficient + # to uniquely identify a panel composed of user-defined + # custom metrics from the DB. + def valid_params?(params) + [ + params[:embedded], + valid_dashboard?(params[:dashboard_path]), + valid_group_title?(params[:group]), + params[:title].present?, + params.has_key?(:y_label) + ].all? + end + + private + + # A group title is valid if it is one of the limited + # options the user can select in the UI. + def valid_group_title?(group) + PrometheusMetricEnums + .custom_group_details + .map { |_, details| details[:group_title] } + .include?(group) + end + + # All custom metrics are displayed on the system dashboard. + # Nil is acceptable as we'll default to the system dashboard. + def valid_dashboard?(dashboard) + dashboard.nil? || SystemDashboardService.system_dashboard?(dashboard) + end + end + + # Returns a new dashboard with only the matching + # metrics from the system dashboard, stripped of + # group info. + # + # Note: This overrides the method #raw_dashboard, + # which means the result will not be cached. This + # is because we are inserting DB info into the + # dashboard before post-processing. This ensures + # we aren't acting on deleted or out-of-date metrics. + # + # @return [Hash] + override :raw_dashboard + def raw_dashboard + panels_not_found!(identifiers) if panels.empty? + + { 'panel_groups' => [{ 'panels' => panels }] } + end + + private + + # Generated dashboard panels for each metric which + # matches the provided input. + # @return [Array<Hash>] + def panels + strong_memoize(:panels) do + metrics.map { |metric| panel_for_metric(metric) } + end + end + + # Metrics which match the provided inputs. + # There may be multiple metrics, but they should be + # displayed in a single panel/chart. + # @return [ActiveRecord::AssociationRelation<PromtheusMetric>] + # rubocop: disable CodeReuse/ActiveRecord + def metrics + project.prometheus_metrics.where( + group: group_key, + title: title, + y_label: y_label + ) + end + # rubocop: enable CodeReuse/ActiveRecord + + # Returns a symbol representing the group that + # the dashboard's group title belongs to. + # It will be one of the keys found under + # PrometheusMetricEnums.custom_groups. + # + # @return [String] + def group_key + strong_memoize(:group_key) do + PrometheusMetricEnums + .group_details + .find { |_, details| details[:group_title] == group } + .first + .to_s + end + end + + # Returns a representation of a PromtheusMetric + # as a dashboard panel. As the panel is generated + # on the fly, we're using default values for info + # not represented in the DB. + # + # @return [Hash] + def panel_for_metric(metric) + { + type: DEFAULT_PANEL_TYPE, + weight: DEFAULT_PANEL_WEIGHT, + title: metric.title, + y_label: metric.y_label, + metrics: [metric.to_metric_hash] + } + end + end + end +end diff --git a/app/services/metrics/dashboard/default_embed_service.rb b/app/services/metrics/dashboard/default_embed_service.rb new file mode 100644 index 00000000000..e1bd98bd5c2 --- /dev/null +++ b/app/services/metrics/dashboard/default_embed_service.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +# Responsible for returning a filtered system dashboard +# containing only the default embedded metrics. This class +# operates by selecting metrics directly from the system +# dashboard. +# +# Why isn't this filtering in a processing stage? By filtering +# here, we ensure the dynamically-determined dashboard is cached. +# +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class DefaultEmbedService < ::Metrics::Dashboard::BaseEmbedService + # For the default filtering for embedded metrics, + # uses the 'id' key in dashboard-yml definition for + # identification. + DEFAULT_EMBEDDED_METRICS_IDENTIFIERS = %w( + system_metrics_kubernetes_container_memory_total + system_metrics_kubernetes_container_cores_total + ).freeze + + # Returns a new dashboard with only the matching + # metrics from the system dashboard, stripped of groups. + # @return [Hash] + def get_raw_dashboard + panels = panel_groups.each_with_object([]) do |group, panels| + matched_panels = group['panels'].select { |panel| matching_panel?(panel) } + + panels.concat(matched_panels) + end + + { 'panel_groups' => [{ 'panels' => panels }] } + end + + private + + # Returns an array of the panels groups on the + # system dashboard + def panel_groups + ::Metrics::Dashboard::SystemDashboardService + .new(project, nil) + .raw_dashboard['panel_groups'] + end + + # Identifies a panel as "matching" if any metric ids in + # the panel is in the list of identifiers to collect. + def matching_panel?(panel) + panel['metrics'].any? do |metric| + metric_identifiers.include?(metric['id']) + end + end + + def metric_identifiers + DEFAULT_EMBEDDED_METRICS_IDENTIFIERS + end + + def identifiers + metric_identifiers.join('|') + end + end + end +end diff --git a/app/services/metrics/dashboard/dynamic_embed_service.rb b/app/services/metrics/dashboard/dynamic_embed_service.rb new file mode 100644 index 00000000000..db5b7c9e32a --- /dev/null +++ b/app/services/metrics/dashboard/dynamic_embed_service.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# Responsible for returning a filtered project dashboard +# containing only the request-provided metrics. The result +# is then cached for future requests. Metrics are identified +# based on a combination of identifiers for now, but the ideal +# would be similar to the approach in DefaultEmbedService, but +# a single unique identifier is not currently available across +# all metric types (custom, project-defined, cluster, or system). +# +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class DynamicEmbedService < ::Metrics::Dashboard::BaseEmbedService + include Gitlab::Utils::StrongMemoize + + class << self + # Determines whether the provided params are sufficient + # to uniquely identify a panel from a yml-defined dashboard. + # + # See https://docs.gitlab.com/ee/user/project/integrations/prometheus.html#defining-custom-dashboards-per-project + # for additional info on defining custom dashboards. + def valid_params?(params) + [ + params[:embedded], + params[:group].present?, + params[:title].present?, + params[:y_label] + ].all? + end + end + + # Returns a new dashboard with only the matching + # metrics from the system dashboard, stripped of groups. + # @return [Hash] + def get_raw_dashboard + not_found! if panels.empty? + + { 'panel_groups' => [{ 'panels' => panels }] } + end + + private + + def panels + strong_memoize(:panels) do + not_found! unless base_dashboard + not_found! unless groups = base_dashboard['panel_groups'] + not_found! unless matching_group = find_group(groups) + not_found! unless all_panels = matching_group['panels'] + + find_panels(all_panels) + end + end + + def base_dashboard + strong_memoize(:base_dashboard) do + Gitlab::Metrics::Dashboard::Finder.find_raw(project, dashboard_path: dashboard_path) + end + end + + def find_group(groups) + groups.find do |candidate_group| + candidate_group['group'] == group + end + end + + def find_panels(all_panels) + all_panels.select do |panel| + panel['title'] == title && panel['y_label'] == y_label + end + end + + def not_found! + panels_not_found!(identifiers) + end + end + end +end diff --git a/app/services/metrics/dashboard/project_dashboard_service.rb b/app/services/metrics/dashboard/project_dashboard_service.rb new file mode 100644 index 00000000000..756d387c0e6 --- /dev/null +++ b/app/services/metrics/dashboard/project_dashboard_service.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class ProjectDashboardService < ::Metrics::Dashboard::BaseService + DASHBOARD_ROOT = ".gitlab/dashboards" + + class << self + def all_dashboard_paths(project) + file_finder(project) + .list_files_for(DASHBOARD_ROOT) + .map do |filepath| + { + path: filepath, + display_name: name_for_path(filepath), + default: false + } + end + end + + def file_finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') + end + + # Grabs the filepath after the base directory. + def name_for_path(filepath) + filepath.delete_prefix("#{DASHBOARD_ROOT}/") + end + end + + private + + # Searches the project repo for a custom-defined dashboard. + def get_raw_dashboard + yml = self.class.file_finder(project).read(dashboard_path) + + YAML.safe_load(yml) + end + + def cache_key + "project_#{project.id}_metrics_dashboard_#{dashboard_path}" + end + end + end +end diff --git a/app/services/metrics/dashboard/system_dashboard_service.rb b/app/services/metrics/dashboard/system_dashboard_service.rb new file mode 100644 index 00000000000..fcd71aadb03 --- /dev/null +++ b/app/services/metrics/dashboard/system_dashboard_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Fetches the system metrics dashboard and formats the output. +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Metrics + module Dashboard + class SystemDashboardService < ::Metrics::Dashboard::BaseService + SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' + SYSTEM_DASHBOARD_NAME = 'Default' + + class << self + def all_dashboard_paths(_project) + [{ + path: SYSTEM_DASHBOARD_PATH, + display_name: SYSTEM_DASHBOARD_NAME, + default: true + }] + end + + def system_dashboard?(filepath) + filepath == SYSTEM_DASHBOARD_PATH + end + end + + private + + def dashboard_path + SYSTEM_DASHBOARD_PATH + end + + # Returns the base metrics shipped with every GitLab service. + def get_raw_dashboard + yml = File.read(Rails.root.join(dashboard_path)) + + YAML.safe_load(yml) + end + + def cache_key + "metrics_dashboard_#{dashboard_path}" + end + + def insert_project_metrics? + true + end + end + end +end diff --git a/app/services/notes/base_service.rb b/app/services/notes/base_service.rb index c1260837c12..b4d04c47cc0 100644 --- a/app/services/notes/base_service.rb +++ b/app/services/notes/base_service.rb @@ -9,5 +9,9 @@ module Notes note.noteable.diffs.clear_cache end end + + def increment_usage_counter(note) + Gitlab::UsageDataCounters::NoteCounter.count(:create, note.noteable_type) + end end end diff --git a/app/services/notes/create_service.rb b/app/services/notes/create_service.rb index 1b46f6d8a72..248e81080cc 100644 --- a/app/services/notes/create_service.rb +++ b/app/services/notes/create_service.rb @@ -21,7 +21,7 @@ module Notes if quick_actions_service.supported?(note) options = { merge_request_diff_head_sha: merge_request_diff_head_sha } - content, update_params = quick_actions_service.execute(note, options) + content, update_params, message = quick_actions_service.execute(note, options) only_commands = content.empty? @@ -41,6 +41,7 @@ module Notes todo_service.new_note(note, current_user) clear_noteable_diffs_cache(note) Suggestions::CreateService.new(note).execute + increment_usage_counter(note) end if quick_actions_service.commands_executed_count.to_i > 0 @@ -52,7 +53,7 @@ module Notes # We must add the error after we call #save because errors are reset # when #save is called if only_commands - note.errors.add(:commands_only, 'Commands applied') + note.errors.add(:commands_only, message.presence || _('Failed to apply commands.')) end end diff --git a/app/services/notes/update_service.rb b/app/services/notes/update_service.rb index 384d1dd2e50..853faed9d85 100644 --- a/app/services/notes/update_service.rb +++ b/app/services/notes/update_service.rb @@ -8,24 +8,70 @@ module Notes old_mentioned_users = note.mentioned_users.to_a note.update(params.merge(updated_by: current_user)) - note.create_new_cross_references!(current_user) - if note.previous_changes.include?('note') - TodoService.new.update_note(note, current_user, old_mentioned_users) + only_commands = false + + quick_actions_service = QuickActionsService.new(project, current_user) + if quick_actions_service.supported?(note) + content, update_params, message = quick_actions_service.execute(note, {}) + + only_commands = content.empty? + + note.note = content + end + + unless only_commands + note.create_new_cross_references!(current_user) + + update_todos(note, old_mentioned_users) + + update_suggestions(note) end - if note.supports_suggestion? - Suggestion.transaction do - note.suggestions.delete_all - Suggestions::CreateService.new(note).execute + if quick_actions_service.commands_executed_count.to_i > 0 + if update_params.present? + quick_actions_service.apply_updates(update_params, note) + note.commands_changes = update_params end - # We need to refresh the previous suggestions call cache - # in order to get the new records. - note.reset + if only_commands + delete_note(note, message) + note = nil + else + note.save + end end note end + + private + + def delete_note(note, message) + # We must add the error after we call #save because errors are reset + # when #save is called + note.errors.add(:commands_only, message.presence || _('Commands did not apply')) + + Notes::DestroyService.new(project, current_user).execute(note) + end + + def update_suggestions(note) + return unless note.supports_suggestion? + + Suggestion.transaction do + note.suggestions.delete_all + Suggestions::CreateService.new(note).execute + end + + # We need to refresh the previous suggestions call cache + # in order to get the new records. + note.reset + end + + def update_todos(note, old_mentioned_users) + return unless note.previous_changes.include?('note') + + TodoService.new.update_note(note, current_user, old_mentioned_users) + end end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 5aa804666f0..83710ffce2f 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -321,6 +321,9 @@ class NotificationService end def decline_project_invite(project_member) + # Must always send, regardless of project/namespace configuration since it's a + # response to the user's action. + mailer.member_invite_declined_email( project_member.real_source_type, project_member.project.id, @@ -351,8 +354,8 @@ class NotificationService end def decline_group_invite(group_member) - # always send this one, since it's a response to the user's own - # action + # Must always send, regardless of project/namespace configuration since it's a + # response to the user's action. mailer.member_invite_declined_email( group_member.real_source_type, @@ -410,6 +413,10 @@ class NotificationService end def pipeline_finished(pipeline, recipients = nil) + # Must always check project configuration since recipients could be a list of emails + # from the PipelinesEmailService integration. + return if pipeline.project.emails_disabled? + email_template = "pipeline_#{pipeline.status}_email" return unless mailer.respond_to?(email_template) @@ -418,7 +425,9 @@ class NotificationService [pipeline.user], :watch, custom_action: :"#{pipeline.status}_pipeline", target: pipeline - ).map(&:notification_email) + ).map do |user| + user.notification_email_for(pipeline.project.group) + end if recipients.any? mailer.public_send(email_template, pipeline, recipients).deliver_later @@ -426,6 +435,8 @@ class NotificationService end def autodevops_disabled(pipeline, recipients) + return if pipeline.project.emails_disabled? + recipients.each do |recipient| mailer.autodevops_disabled_email(pipeline, recipient).deliver_later end @@ -470,10 +481,14 @@ class NotificationService end def repository_cleanup_success(project, user) + return if project.emails_disabled? + mailer.send(:repository_cleanup_success_email, project, user).deliver_later end def repository_cleanup_failure(project, user, error) + return if project.emails_disabled? + mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later end @@ -593,7 +608,7 @@ class NotificationService end def deliver_access_request_email(recipient, member) - mailer.member_access_requested_email(member.real_source_type, member.id, recipient.user.notification_email).deliver_later + mailer.member_access_requested_email(member.real_source_type, member.id, recipient.user.id).deliver_later end def fallback_to_group_owners_maintainers?(recipients, member) diff --git a/app/services/projects/after_import_service.rb b/app/services/projects/after_import_service.rb index bbdde4408d2..e30da0f26df 100644 --- a/app/services/projects/after_import_service.rb +++ b/app/services/projects/after_import_service.rb @@ -13,7 +13,7 @@ module Projects repository.delete_all_refs_except(RESERVED_REF_PREFIXES) end rescue Projects::HousekeepingService::LeaseTaken => e - Rails.logger.info( + Rails.logger.info( # rubocop:disable Gitlab/RailsLogger "Could not perform housekeeping for project #{@project.full_path} (#{@project.id}): #{e}") end diff --git a/app/services/projects/base_move_relations_service.rb b/app/services/projects/base_move_relations_service.rb index 24dec1f3a45..3a159cef58b 100644 --- a/app/services/projects/base_move_relations_service.rb +++ b/app/services/projects/base_move_relations_service.rb @@ -10,17 +10,5 @@ module Projects true end - - private - - # rubocop: disable CodeReuse/ActiveRecord - def prepare_relation(relation, id_param = :id) - if Gitlab::Database.postgresql? - relation - else - relation.model.where("#{id_param}": relation.pluck(id_param)) - end - end - # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 9f335cceb67..89dc4375c63 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -151,7 +151,7 @@ module Projects log_message = message.dup log_message << " Project ID: #{@project.id}" if @project&.id - Rails.logger.error(log_message) + Rails.logger.error(log_message) # rubocop:disable Gitlab/RailsLogger if @project && @project.persisted? && @project.import_state @project.import_state.mark_as_failed(message) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index d8e670e40ce..5893b8eedff 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -18,7 +18,7 @@ module Projects schedule_stale_repos_removal job_id = ProjectDestroyWorker.perform_async(project.id, current_user.id, params) - Rails.logger.info("User #{current_user.id} scheduled destruction of project #{project.full_path} with job ID #{job_id}") + Rails.logger.info("User #{current_user.id} scheduled destruction of project #{project.full_path} with job ID #{job_id}") # rubocop:disable Gitlab/RailsLogger end def execute @@ -173,6 +173,7 @@ module Projects end def remove_registry_tags + return true unless Gitlab.config.registry.enabled return false unless remove_legacy_registry_tags project.container_repositories.find_each do |container_repository| @@ -210,11 +211,20 @@ module Projects end def flush_caches(project) - project.repository.before_delete + ignore_git_errors(repo_path) { project.repository.before_delete } - Repository.new(wiki_path, project, disk_path: repo_path).before_delete + ignore_git_errors(wiki_path) { Repository.new(wiki_path, project, disk_path: repo_path).before_delete } Projects::ForksCountService.new(project).delete_cache end + + # If we get a Gitaly error, the repository may be corrupted. We can + # ignore these errors since we're going to trash the repositories + # anyway. + def ignore_git_errors(disk_path, &block) + yield + rescue Gitlab::Git::CommandError => e + Gitlab::GitLogger.warn(class: self.class.name, project_id: project.id, disk_path: disk_path, message: e.to_s) + end end end diff --git a/app/services/projects/fetch_statistics_increment_service.rb b/app/services/projects/fetch_statistics_increment_service.rb index 8644e6bf313..b150fd2d9f1 100644 --- a/app/services/projects/fetch_statistics_increment_service.rb +++ b/app/services/projects/fetch_statistics_increment_service.rb @@ -12,14 +12,9 @@ module Projects increment_fetch_count_sql = <<~SQL INSERT INTO #{table_name} (project_id, date, fetch_count) VALUES (#{project.id}, '#{Date.today}', 1) + ON CONFLICT (project_id, date) DO UPDATE SET fetch_count = #{table_name}.fetch_count + 1 SQL - increment_fetch_count_sql += if Gitlab::Database.postgresql? - "ON CONFLICT (project_id, date) DO UPDATE SET fetch_count = #{table_name}.fetch_count + 1" - else - "ON DUPLICATE KEY UPDATE fetch_count = #{table_name}.fetch_count + 1" - end - ActiveRecord::Base.connection.execute(increment_fetch_count_sql) end diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb index 3d0b8f58612..affe6e5668d 100644 --- a/app/services/projects/hashed_storage/migrate_attachments_service.rb +++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb @@ -5,7 +5,7 @@ module Projects class MigrateAttachmentsService < BaseAttachmentService def initialize(project, old_disk_path, logger: nil) @project = project - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger @old_disk_path = old_disk_path @skipped = false end diff --git a/app/services/projects/hashed_storage/rollback_attachments_service.rb b/app/services/projects/hashed_storage/rollback_attachments_service.rb index 5c6b92f965c..fb09eaa4586 100644 --- a/app/services/projects/hashed_storage/rollback_attachments_service.rb +++ b/app/services/projects/hashed_storage/rollback_attachments_service.rb @@ -5,7 +5,7 @@ module Projects class RollbackAttachmentsService < BaseAttachmentService def initialize(project, logger: nil) @project = project - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger @old_disk_path = project.disk_path end diff --git a/app/services/projects/hashed_storage/rollback_service.rb b/app/services/projects/hashed_storage/rollback_service.rb index 25767f5de5e..ee41aae64a5 100644 --- a/app/services/projects/hashed_storage/rollback_service.rb +++ b/app/services/projects/hashed_storage/rollback_service.rb @@ -8,7 +8,7 @@ module Projects def initialize(project, old_disk_path, logger: nil) @project = project @old_disk_path = old_disk_path - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def execute diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index e3491282a8a..9c6d7ef41f6 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -62,7 +62,7 @@ module Projects end def cleanup_and_notify_error - Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") + Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger FileUtils.rm_rf(@shared.export_path) @@ -76,7 +76,7 @@ module Projects end def notify_success - Rails.logger.info("Import/Export - Project #{project.name} with ID: #{project.id} successfully exported") + Rails.logger.info("Import/Export - Project #{project.name} with ID: #{project.id} successfully exported") # rubocop:disable Gitlab/RailsLogger end def notify_error diff --git a/app/services/projects/move_deploy_keys_projects_service.rb b/app/services/projects/move_deploy_keys_projects_service.rb index b6a3af8c7b8..01419563538 100644 --- a/app/services/projects/move_deploy_keys_projects_service.rb +++ b/app/services/projects/move_deploy_keys_projects_service.rb @@ -16,8 +16,7 @@ module Projects private def move_deploy_keys_projects - prepare_relation(non_existent_deploy_keys_projects) - .update_all(project_id: @project.id) + non_existent_deploy_keys_projects.update_all(project_id: @project.id) end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/services/projects/move_lfs_objects_projects_service.rb b/app/services/projects/move_lfs_objects_projects_service.rb index 308a54ad06e..10e19014db4 100644 --- a/app/services/projects/move_lfs_objects_projects_service.rb +++ b/app/services/projects/move_lfs_objects_projects_service.rb @@ -16,8 +16,7 @@ module Projects private def move_lfs_objects_projects - prepare_relation(non_existent_lfs_objects_projects) - .update_all(project_id: @project.lfs_storage_project.id) + non_existent_lfs_objects_projects.update_all(project_id: @project.lfs_storage_project.id) end def remove_remaining_lfs_objects_project diff --git a/app/services/projects/move_notification_settings_service.rb b/app/services/projects/move_notification_settings_service.rb index e740c44bd26..65a888fe26b 100644 --- a/app/services/projects/move_notification_settings_service.rb +++ b/app/services/projects/move_notification_settings_service.rb @@ -16,8 +16,7 @@ module Projects private def move_notification_settings - prepare_relation(non_existent_notifications) - .update_all(source_id: @project.id) + non_existent_notifications.update_all(source_id: @project.id) end # Remove remaining notification settings from source_project diff --git a/app/services/projects/move_project_authorizations_service.rb b/app/services/projects/move_project_authorizations_service.rb index 2985ba89014..c95ad60ab5e 100644 --- a/app/services/projects/move_project_authorizations_service.rb +++ b/app/services/projects/move_project_authorizations_service.rb @@ -21,8 +21,7 @@ module Projects private def move_project_authorizations - prepare_relation(non_existent_authorization, :user_id) - .update_all(project_id: @project.id) + non_existent_authorization.update_all(project_id: @project.id) end def remove_remaining_authorizations diff --git a/app/services/projects/move_project_group_links_service.rb b/app/services/projects/move_project_group_links_service.rb index cf4b291c761..d1aa9af2bcb 100644 --- a/app/services/projects/move_project_group_links_service.rb +++ b/app/services/projects/move_project_group_links_service.rb @@ -20,8 +20,7 @@ module Projects private def move_group_links - prepare_relation(non_existent_group_links) - .update_all(project_id: @project.id) + non_existent_group_links.update_all(project_id: @project.id) end # Remove remaining project group links from source_project diff --git a/app/services/projects/move_project_members_service.rb b/app/services/projects/move_project_members_service.rb index faf389241d2..de4e7e5a1e3 100644 --- a/app/services/projects/move_project_members_service.rb +++ b/app/services/projects/move_project_members_service.rb @@ -20,7 +20,7 @@ module Projects private def move_project_members - prepare_relation(non_existent_members).update_all(source_id: @project.id) + non_existent_members.update_all(source_id: @project.id) end def remove_remaining_members diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb index a25c985585b..64f9b611c40 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_service_template.rb @@ -15,7 +15,7 @@ module Projects def propagate return unless @template.active? - Rails.logger.info("Propagating services for template #{@template.id}") + Rails.logger.info("Propagating services for template #{@template.id}") # rubocop:disable Gitlab/RailsLogger propagate_projects_with_template end diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 1244a0f72a7..13a467a3ef9 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -2,31 +2,52 @@ module Projects class UpdateRemoteMirrorService < BaseService - attr_reader :errors + MAX_TRIES = 3 - def execute(remote_mirror) + def execute(remote_mirror, tries) return success unless remote_mirror.enabled? - errors = [] + update_mirror(remote_mirror) - begin - remote_mirror.ensure_remote! - repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true) + success + rescue Gitlab::Git::CommandError => e + # This happens if one of the gitaly calls above fail, for example when + # branches have diverged, or the pre-receive hook fails. + retry_or_fail(remote_mirror, e.message, tries) - opts = {} - if remote_mirror.only_protected_branches? - opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) - end + error(e.message) + rescue => e + remote_mirror.mark_as_failed!(e.message) + raise e + end + + private + + def update_mirror(remote_mirror) + remote_mirror.update_start! + + remote_mirror.ensure_remote! + repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true) - remote_mirror.update_repository(opts) - rescue => e - errors << e.message.strip + opts = {} + if remote_mirror.only_protected_branches? + opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) end - if errors.present? - error(errors.join("\n\n")) + remote_mirror.update_repository(opts) + + remote_mirror.update_finish! + end + + def retry_or_fail(mirror, message, tries) + if tries < MAX_TRIES + mirror.mark_for_retry!(message) else - success + # It's not likely we'll be able to recover from this ourselves, so we'll + # notify the users of the problem, and don't trigger any sidekiq retries + # Instead, we'll wait for the next change to try the push again, or until + # a user manually retries. + mirror.mark_as_failed!(message) end end end diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index 2bc04470342..8acbdc7e02b 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -9,6 +9,7 @@ module Projects # rubocop: disable CodeReuse/ActiveRecord def execute + remove_unallowed_params validate! ensure_wiki_exists if enabling_wiki? @@ -54,6 +55,10 @@ module Projects end end + def remove_unallowed_params + params.delete(:emails_disabled) unless can?(current_user, :set_emails_disabled, project) + end + def after_update todos_features_changes = %w( issues_access_level @@ -122,7 +127,7 @@ module Projects ProjectWiki.new(project, project.owner).wiki rescue ProjectWiki::CouldNotCreateWikiError log_error("Could not create wiki for #{project.full_name}") - Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki') + Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki').increment end def update_pages_config diff --git a/app/services/projects/update_statistics_service.rb b/app/services/projects/update_statistics_service.rb index 28677a398f3..cc6ffa9eafc 100644 --- a/app/services/projects/update_statistics_service.rb +++ b/app/services/projects/update_statistics_service.rb @@ -5,7 +5,7 @@ module Projects def execute return unless project - Rails.logger.info("Updating statistics for project #{project.id}") + Rails.logger.info("Updating statistics for project #{project.id}") # rubocop:disable Gitlab/RailsLogger project.statistics.refresh!(only: statistics.map(&:to_sym)) end diff --git a/app/services/prometheus/proxy_service.rb b/app/services/prometheus/proxy_service.rb index c5d2b84878b..a62eb76b8ce 100644 --- a/app/services/prometheus/proxy_service.rb +++ b/app/services/prometheus/proxy_service.rb @@ -98,7 +98,7 @@ module Prometheus end def prometheus_client_wrapper - prometheus_adapter&.prometheus_client_wrapper + prometheus_adapter&.prometheus_client end def can_query? diff --git a/app/services/quick_actions/interpret_service.rb b/app/services/quick_actions/interpret_service.rb index 8ff73522e5f..7f944e25887 100644 --- a/app/services/quick_actions/interpret_service.rb +++ b/app/services/quick_actions/interpret_service.rb @@ -31,17 +31,19 @@ module QuickActions end # Takes a text and interprets the commands that are extracted from it. - # Returns the content without commands, and hash of changes to be applied to a record. + # Returns the content without commands, a hash of changes to be applied to a record + # and a string containing the execution_message to show to the user. def execute(content, quick_action_target, only: nil) - return [content, {}] unless current_user.can?(:use_quick_actions) + return [content, {}, ''] unless current_user.can?(:use_quick_actions) @quick_action_target = quick_action_target @updates = {} + @execution_message = {} content, commands = extractor.extract_commands(content, only: only) extract_updates(commands) - [content, @updates] + [content, @updates, execution_messages_for(commands)] end # Takes a text and interprets the commands that are extracted from it. @@ -119,8 +121,12 @@ module QuickActions labels_params.scan(/"([^"]+)"|([^ ]+)/).flatten.compact end - def find_label_references(labels_param) - find_labels(labels_param).map(&:to_reference) + def find_label_references(labels_param, format = :id) + labels_to_reference(find_labels(labels_param), format) + end + + def labels_to_reference(labels, format = :id) + labels.map { |l| l.to_reference(format: format) } end def find_label_ids(labels_param) @@ -128,11 +134,24 @@ module QuickActions end def explain_commands(commands) + map_commands(commands, :explain) + end + + def execution_messages_for(commands) + map_commands(commands, :execute_message).join(' ') + end + + def map_commands(commands, method) commands.map do |name, arg| definition = self.class.definition_by_name(name) next unless definition - definition.explain(self, arg) + case method + when :explain + definition.explain(self, arg) + when :execute_message + @execution_message[name.to_sym] || definition.execute_message(self, arg) + end end.compact end diff --git a/app/services/submit_usage_ping_service.rb b/app/services/submit_usage_ping_service.rb index 62222d3fd2a..4f10f220298 100644 --- a/app/services/submit_usage_ping_service.rb +++ b/app/services/submit_usage_ping_service.rb @@ -28,7 +28,7 @@ class SubmitUsagePingService true rescue Gitlab::HTTP::Error => e - Rails.logger.info "Unable to contact GitLab, Inc.: #{e}" + Rails.logger.info "Unable to contact GitLab, Inc.: #{e}" # rubocop:disable Gitlab/RailsLogger false end diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index e4564bc9b00..ee7223d6349 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -597,6 +597,14 @@ module SystemNoteService note_text =~ /\A#{cross_reference_note_prefix}/i end + def zoom_link_added(issue, project, author) + create_note(NoteSummary.new(issue, project, author, _('added a Zoom call to this issue'), action: 'pinned_embed')) + end + + def zoom_link_removed(issue, project, author) + create_note(NoteSummary.new(issue, project, author, _('removed a Zoom call from this issue'), action: 'pinned_embed')) + end + private # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/services/update_deployment_service.rb b/app/services/update_deployment_service.rb index 49a7d0178f4..dcafebae52d 100644 --- a/app/services/update_deployment_service.rb +++ b/app/services/update_deployment_service.rb @@ -42,7 +42,7 @@ class UpdateDeploymentService return unless environment_url @expanded_environment_url = - ExpandVariables.expand(environment_url, variables) + ExpandVariables.expand(environment_url, -> { variables }) end def environment_url diff --git a/app/services/update_snippet_service.rb b/app/services/update_snippet_service.rb index 15bc1046a4e..2969c360de5 100644 --- a/app/services/update_snippet_service.rb +++ b/app/services/update_snippet_service.rb @@ -25,6 +25,8 @@ class UpdateSnippetService < BaseService snippet.assign_attributes(params) spam_check(snippet, current_user) - snippet.save + snippet.save.tap do |succeeded| + Gitlab::UsageDataCounters::SnippetCounter.count(:update) if succeeded + end end end diff --git a/app/services/users/refresh_authorized_projects_service.rb b/app/services/users/refresh_authorized_projects_service.rb index 4a26d2be2af..ae67b4f5256 100644 --- a/app/services/users/refresh_authorized_projects_service.rb +++ b/app/services/users/refresh_authorized_projects_service.rb @@ -102,13 +102,7 @@ module Users end def fresh_authorizations - klass = if Group.supports_nested_objects? - Gitlab::ProjectAuthorizations::WithNestedGroups - else - Gitlab::ProjectAuthorizations::WithoutNestedGroups - end - - klass.new(user).calculate + Gitlab::ProjectAuthorizations.new(user).calculate end end end diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index 1fee8bfcd31..8c294218708 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -17,8 +17,10 @@ class WebHookService @hook = hook @data = data @hook_name = hook_name.to_s - @request_options = { timeout: Gitlab.config.gitlab.webhook_timeout } - @request_options.merge!(allow_local_requests: true) if @hook.is_a?(SystemHook) + @request_options = { + timeout: Gitlab.config.gitlab.webhook_timeout, + allow_local_requests: hook.allow_local_requests? + } end def execute @@ -53,7 +55,7 @@ class WebHookService error_message: e.to_s ) - Rails.logger.error("WebHook Error => #{e}") + Rails.logger.error("WebHook Error => #{e}") # rubocop:disable Gitlab/RailsLogger { status: :error, diff --git a/app/services/wiki_pages/base_service.rb b/app/services/wiki_pages/base_service.rb index e259f5bd1bc..b9df690c2b7 100644 --- a/app/services/wiki_pages/base_service.rb +++ b/app/services/wiki_pages/base_service.rb @@ -8,6 +8,12 @@ module WikiPages page_data = Gitlab::DataBuilder::WikiPage.build(page, current_user, action) @project.execute_hooks(page_data, :wiki_page_hooks) @project.execute_services(page_data, :wiki_page_hooks) + increment_usage(action) + end + + # This method throws an error if the action is an unanticipated value. + def increment_usage(action) + Gitlab::UsageDataCounters::WikiPageCounter.count(action) end end end diff --git a/app/services/zoom_notes_service.rb b/app/services/zoom_notes_service.rb new file mode 100644 index 00000000000..983a7fcacd1 --- /dev/null +++ b/app/services/zoom_notes_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class ZoomNotesService + def initialize(issue, project, current_user, old_description: nil) + @issue = issue + @project = project + @current_user = current_user + @old_description = old_description + end + + def execute + return if @issue.description == @old_description + + if zoom_link_added? + zoom_link_added_notification + elsif zoom_link_removed? + zoom_link_removed_notification + end + end + + private + + def zoom_link_added? + has_zoom_link?(@issue.description) && !has_zoom_link?(@old_description) + end + + def zoom_link_removed? + !has_zoom_link?(@issue.description) && has_zoom_link?(@old_description) + end + + def has_zoom_link?(text) + Gitlab::ZoomLinkExtractor.new(text).match? + end + + def zoom_link_added_notification + SystemNoteService.zoom_link_added(@issue, @project, @current_user) + end + + def zoom_link_removed_notification + SystemNoteService.zoom_link_removed(@issue, @project, @current_user) + end +end |