diff options
Diffstat (limited to 'app/services')
86 files changed, 1331 insertions, 643 deletions
diff --git a/app/services/alert_management/http_integrations/create_service.rb b/app/services/alert_management/http_integrations/create_service.rb index 576e38c23aa..e7f1084ce5c 100644 --- a/app/services/alert_management/http_integrations/create_service.rb +++ b/app/services/alert_management/http_integrations/create_service.rb @@ -9,14 +9,14 @@ module AlertManagement def initialize(project, current_user, params) @project = project @current_user = current_user - @params = params + @params = params.with_indifferent_access end def execute return error_no_permissions unless allowed? return error_multiple_integrations unless creation_allowed? - integration = project.alert_management_http_integrations.create(params) + integration = project.alert_management_http_integrations.create(permitted_params) return error_in_create(integration) unless integration.valid? success(integration) @@ -34,6 +34,15 @@ module AlertManagement project.alert_management_http_integrations.empty? end + def permitted_params + params.slice(*permitted_params_keys) + end + + # overriden in EE + def permitted_params_keys + %i[name active] + end + def error(message) ServiceResponse.error(message: message) end diff --git a/app/services/alert_management/sync_alert_service_data_service.rb b/app/services/alert_management/sync_alert_service_data_service.rb deleted file mode 100644 index 1ba197065c5..00000000000 --- a/app/services/alert_management/sync_alert_service_data_service.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module AlertManagement - class SyncAlertServiceDataService - # @param alert_service [AlertsService] - def initialize(alert_service) - @alert_service = alert_service - end - - def execute - http_integration = find_http_integration - - result = if http_integration - update_integration_data(http_integration) - else - create_integration - end - - result ? ServiceResponse.success : ServiceResponse.error(message: 'Update failed') - end - - private - - attr_reader :alert_service - - def find_http_integration - AlertManagement::HttpIntegrationsFinder.new( - alert_service.project, - endpoint_identifier: ::AlertManagement::HttpIntegration::LEGACY_IDENTIFIER - ) - .execute - .first - end - - def create_integration - new_integration = AlertManagement::HttpIntegration.create( - project_id: alert_service.project_id, - name: 'HTTP endpoint', - endpoint_identifier: AlertManagement::HttpIntegration::LEGACY_IDENTIFIER, - active: alert_service.active, - encrypted_token: alert_service.data.encrypted_token, - encrypted_token_iv: alert_service.data.encrypted_token_iv - ) - - new_integration.persisted? - end - - def update_integration_data(http_integration) - http_integration.update( - active: alert_service.active, - encrypted_token: alert_service.data.encrypted_token, - encrypted_token_iv: alert_service.data.encrypted_token_iv - ) - end - end -end diff --git a/app/services/boards/base_items_list_service.rb b/app/services/boards/base_items_list_service.rb new file mode 100644 index 00000000000..851120ef597 --- /dev/null +++ b/app/services/boards/base_items_list_service.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Boards + class BaseItemsListService < Boards::BaseService + include Gitlab::Utils::StrongMemoize + include ActiveRecord::ConnectionAdapters::Quoting + + def execute + return items.order_closed_date_desc if list&.closed? + + ordered_items + end + + private + + def ordered_items + raise NotImplementedError + end + + def finder + raise NotImplementedError + end + + def board + raise NotImplementedError + end + + def item_model + raise NotImplementedError + end + + # We memoize the query here since the finder methods we use are quite complex. This does not memoize the result of the query. + # rubocop: disable CodeReuse/ActiveRecord + def items + strong_memoize(:items) do + filter(finder.execute).reorder(nil) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def filter(items) + # when grouping board issues by epics (used in board swimlanes) + # we need to get all issues in the board + # TODO: ignore hidden columns - + # https://gitlab.com/gitlab-org/gitlab/-/issues/233870 + return items if params[:all_lists] + + items = without_board_labels(items) unless list&.movable? || list&.closed? + items = with_list_label(items) if list&.label? + items + end + + def list + return unless params.key?(:id) + + strong_memoize(:list) do + id = params[:id] + + if board.lists.loaded? + board.lists.find { |l| l.id == id } + else + board.lists.find(id) + end + end + end + + def filter_params + set_parent + set_state + set_attempt_search_optimizations + + params + end + + def set_parent + if parent.is_a?(Group) + params[:group_id] = parent.id + else + params[:project_id] = parent.id + end + end + + def set_state + return if params[:all_lists] + + params[:state] = list && list.closed? ? 'closed' : 'opened' + end + + def set_attempt_search_optimizations + return unless params[:search].present? + + if board.group_board? + params[:attempt_group_search_optimizations] = true + else + params[:attempt_project_search_optimizations] = true + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def board_label_ids + @board_label_ids ||= board.lists.movable.pluck(:label_id) + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def without_board_labels(items) + return items unless board_label_ids.any? + + items.where.not('EXISTS (?)', label_links(board_label_ids).limit(1)) + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def label_links(label_ids) + LabelLink + .where('label_links.target_type = ?', item_model) + .where(item_model.arel_table[:id].eq(LabelLink.arel_table[:target_id]).to_sql) + .where(label_id: label_ids) + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def with_list_label(items) + items.where('EXISTS (?)', label_links(list.label_id).limit(1)) + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/app/services/boards/issues/list_service.rb b/app/services/boards/issues/list_service.rb index ab9d11abe98..27d59e052c7 100644 --- a/app/services/boards/issues/list_service.rb +++ b/app/services/boards/issues/list_service.rb @@ -2,26 +2,20 @@ module Boards module Issues - class ListService < Boards::BaseService + class ListService < Boards::BaseItemsListService include Gitlab::Utils::StrongMemoize def self.valid_params IssuesFinder.valid_params end - def execute - return fetch_issues.order_closed_date_desc if list&.closed? - - fetch_issues.order_by_position_and_priority(with_cte: params[:search].present?) - end - # rubocop: disable CodeReuse/ActiveRecord def metadata issues = Issue.arel_table keys = metadata_fields.keys # TODO: eliminate need for SQL literal fragment columns = Arel.sql(metadata_fields.values_at(*keys).join(', ')) - results = Issue.where(id: fetch_issues.select(issues[:id])).pluck(columns) + results = Issue.where(id: items.select(issues[:id])).pluck(columns) Hash[keys.zip(results.flatten)] end @@ -29,74 +23,28 @@ module Boards private - def metadata_fields - { size: 'COUNT(*)' } - end - - # We memoize the query here since the finder methods we use are quite complex. This does not memoize the result of the query. - # rubocop: disable CodeReuse/ActiveRecord - def fetch_issues - strong_memoize(:fetch_issues) do - issues = IssuesFinder.new(current_user, filter_params).execute - - filter(issues).reorder(nil) - end + def ordered_items + items.order_by_position_and_priority(with_cte: params[:search].present?) end - # rubocop: enable CodeReuse/ActiveRecord - def filter(issues) - # when grouping board issues by epics (used in board swimlanes) - # we need to get all issues in the board - # TODO: ignore hidden columns - - # https://gitlab.com/gitlab-org/gitlab/-/issues/233870 - return issues if params[:all_lists] - - issues = without_board_labels(issues) unless list&.movable? || list&.closed? - issues = with_list_label(issues) if list&.label? - issues + def finder + IssuesFinder.new(current_user, filter_params) end def board @board ||= parent.boards.find(params[:board_id]) end - def list - return unless params.key?(:id) - - strong_memoize(:list) do - id = params[:id] - - if board.lists.loaded? - board.lists.find { |l| l.id == id } - else - board.lists.find(id) - end - end + def metadata_fields + { size: 'COUNT(*)' } end def filter_params - set_parent - set_state set_scope set_non_archived - set_attempt_search_optimizations set_issue_types - params - end - - def set_parent - if parent.is_a?(Group) - params[:group_id] = parent.id - else - params[:project_id] = parent.id - end - end - - def set_state - return if params[:all_lists] - - params[:state] = list && list.closed? ? 'closed' : 'opened' + super end def set_scope @@ -107,49 +55,12 @@ module Boards params[:non_archived] = parent.is_a?(Group) end - def set_attempt_search_optimizations - return unless params[:search].present? - - if board.group_board? - params[:attempt_group_search_optimizations] = true - else - params[:attempt_project_search_optimizations] = true - end - end - def set_issue_types params[:issue_types] = Issue::TYPES_FOR_LIST end - # rubocop: disable CodeReuse/ActiveRecord - def board_label_ids - @board_label_ids ||= board.lists.movable.pluck(:label_id) - end - # rubocop: enable CodeReuse/ActiveRecord - - # rubocop: disable CodeReuse/ActiveRecord - def without_board_labels(issues) - return issues unless board_label_ids.any? - - issues.where.not('EXISTS (?)', issues_label_links.limit(1)) - end - # rubocop: enable CodeReuse/ActiveRecord - - # rubocop: disable CodeReuse/ActiveRecord - def issues_label_links - LabelLink.where("label_links.target_type = 'Issue' AND label_links.target_id = issues.id").where(label_id: board_label_ids) - end - # rubocop: enable CodeReuse/ActiveRecord - - # rubocop: disable CodeReuse/ActiveRecord - def with_list_label(issues) - issues.where('EXISTS (?)', LabelLink.where("label_links.target_type = 'Issue' AND label_links.target_id = issues.id") - .where("label_links.label_id = ?", list.label_id).limit(1)) - end - # rubocop: enable CodeReuse/ActiveRecord - - def board_group - board.group_board? ? parent : parent.group + def item_model + Issue end end end diff --git a/app/services/bulk_create_integration_service.rb b/app/services/bulk_create_integration_service.rb index 61c5565db60..df78c3645c7 100644 --- a/app/services/bulk_create_integration_service.rb +++ b/app/services/bulk_create_integration_service.rb @@ -38,10 +38,6 @@ class BulkCreateIntegrationService if integration.external_issue_tracker? Project.where(id: batch.select(:id)).update_all(has_external_issue_tracker: true) end - - if integration.external_wiki? - Project.where(id: batch.select(:id)).update_all(has_external_wiki: true) - end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb index 86d0cf079fc..629d85b041f 100644 --- a/app/services/ci/create_downstream_pipeline_service.rb +++ b/app/services/ci/create_downstream_pipeline_service.rb @@ -33,9 +33,7 @@ module Ci pipeline_params.fetch(:target_revision)) downstream_pipeline = service.execute( - pipeline_params.fetch(:source), **pipeline_params[:execute_params]) do |pipeline| - pipeline.variables.build(@bridge.downstream_variables) - end + pipeline_params.fetch(:source), **pipeline_params[:execute_params]) downstream_pipeline.tap do |pipeline| update_bridge_status!(@bridge, pipeline) diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index dbe81521cfc..d3001e54288 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -27,6 +27,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Limit::JobActivity, Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines, Gitlab::Ci::Pipeline::Chain::Metrics, + Gitlab::Ci::Pipeline::Chain::TemplateUsage, Gitlab::Ci::Pipeline::Chain::Pipeline::Process].freeze # Create a new pipeline in the specified project. @@ -81,7 +82,11 @@ module Ci .new(pipeline, command, SEQUENCE) .build! - schedule_head_pipeline_update if pipeline.persisted? + if pipeline.persisted? + schedule_head_pipeline_update + record_conversion_event + create_namespace_onboarding_action + end # If pipeline is not persisted, try to recover IID pipeline.reset_project_iid unless pipeline.persisted? @@ -116,6 +121,15 @@ module Ci end end + def record_conversion_event + Experiments::RecordConversionEventWorker.perform_async(:ci_syntax_templates, current_user.id) + Experiments::RecordConversionEventWorker.perform_async(:pipelines_empty_state, current_user.id) + end + + def create_namespace_onboarding_action + Namespaces::OnboardingPipelineCreatedWorker.perform_async(project.namespace_id) + end + def extra_options(content: nil, dry_run: false) { content: content, dry_run: dry_run } end diff --git a/app/services/ci/create_web_ide_terminal_service.rb b/app/services/ci/create_web_ide_terminal_service.rb index a78281aed16..785d82094b9 100644 --- a/app/services/ci/create_web_ide_terminal_service.rb +++ b/app/services/ci/create_web_ide_terminal_service.rb @@ -6,7 +6,7 @@ module Ci TerminalCreationError = Class.new(StandardError) - TERMINAL_NAME = 'terminal'.freeze + TERMINAL_NAME = 'terminal' attr_reader :terminal diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb index 6e7caba8545..7d8a3c17abe 100644 --- a/app/services/ci/destroy_expired_job_artifacts_service.rb +++ b/app/services/ci/destroy_expired_job_artifacts_service.rb @@ -12,6 +12,10 @@ module Ci EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock' LOCK_TIMEOUT = 6.minutes + def initialize + @removed_artifacts_count = 0 + end + ## # Destroy expired job artifacts on GitLab instance # @@ -20,40 +24,22 @@ module Ci # which is scheduled every 7 minutes. def execute in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do - loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do - destroy_artifacts_batch - end + destroy_job_artifacts_with_slow_iteration(Time.current) end - end - - private - def destroy_artifacts_batch - destroy_job_artifacts_batch || destroy_pipeline_artifacts_batch + @removed_artifacts_count end - def destroy_job_artifacts_batch - artifacts = Ci::JobArtifact - .expired(BATCH_SIZE) - .unlocked - .with_destroy_preloads - .to_a - - return false if artifacts.empty? - - parallel_destroy_batch(artifacts) - true - end - - # TODO: Make sure this can also be parallelized - # https://gitlab.com/gitlab-org/gitlab/-/issues/270973 - def destroy_pipeline_artifacts_batch - artifacts = Ci::PipelineArtifact.expired(BATCH_SIZE).to_a - return false if artifacts.empty? + private - artifacts.each(&:destroy!) + def destroy_job_artifacts_with_slow_iteration(start_at) + Ci::JobArtifact.expired_before(start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index| + artifacts = relation.unlocked.with_destroy_preloads.to_a - true + parallel_destroy_batch(artifacts) if artifacts.any? + break if loop_timeout?(start_at) + break if index >= LOOP_LIMIT + end end def parallel_destroy_batch(job_artifacts) @@ -64,14 +50,14 @@ module Ci end # This is executed outside of the transaction because it depends on Redis - update_statistics_for(job_artifacts) - destroyed_artifacts_counter.increment({}, job_artifacts.size) + update_project_statistics_for(job_artifacts) + increment_monitoring_statistics(job_artifacts.size) end # This method is implemented in EE and it must do only database work def destroy_related_records_for(job_artifacts); end - def update_statistics_for(job_artifacts) + def update_project_statistics_for(job_artifacts) artifacts_by_project = job_artifacts.group_by(&:project) artifacts_by_project.each do |project, artifacts| delta = -artifacts.sum { |artifact| artifact.size.to_i } @@ -80,6 +66,11 @@ module Ci end end + def increment_monitoring_statistics(size) + destroyed_artifacts_counter.increment({}, size) + @removed_artifacts_count += size + end + def destroyed_artifacts_counter strong_memoize(:destroyed_artifacts_counter) do name = :destroyed_job_artifacts_count_total @@ -88,6 +79,10 @@ module Ci ::Gitlab::Metrics.counter(name, comment) end end + + def loop_timeout?(start_at) + Time.current > start_at + LOOP_TIMEOUT + end end end diff --git a/app/services/ci/pipelines/create_artifact_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb index bfaf317241a..9f5c445c91a 100644 --- a/app/services/ci/pipelines/create_artifact_service.rb +++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - module Pipelines - class CreateArtifactService + module PipelineArtifacts + class CoverageReportService def execute(pipeline) return unless pipeline.can_generate_coverage_reports? return if pipeline.has_coverage_reports? diff --git a/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb new file mode 100644 index 00000000000..0dbabe178da --- /dev/null +++ b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Ci + module PipelineArtifacts + class DestroyExpiredArtifactsService + include ::Gitlab::LoopHelpers + include ::Gitlab::Utils::StrongMemoize + + BATCH_SIZE = 100 + LOOP_TIMEOUT = 5.minutes + LOOP_LIMIT = 1000 + + def initialize + @removed_artifacts_count = 0 + end + + def execute + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + destroy_artifacts_batch + end + + @removed_artifacts_count + end + + private + + def destroy_artifacts_batch + artifacts = ::Ci::PipelineArtifact.expired(BATCH_SIZE).to_a + return false if artifacts.empty? + + artifacts.each(&:destroy!) + increment_stats(artifacts.size) + + true + end + + def increment_stats(size) + destroyed_artifacts_counter.increment({}, size) + @removed_artifacts_count += size + end + + def destroyed_artifacts_counter + strong_memoize(:destroyed_artifacts_counter) do + name = :destroyed_pipeline_artifacts_count_total + comment = 'Counter of destroyed expired pipeline artifacts' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end +end diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb index d9f41b7040e..a31f5e9056e 100644 --- a/app/services/ci/pipeline_trigger_service.rb +++ b/app/services/ci/pipeline_trigger_service.rb @@ -21,10 +21,10 @@ module Ci # this check is to not leak the presence of the project if user cannot read it return unless trigger.project == project - pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: params[:ref]) + pipeline = Ci::CreatePipelineService + .new(project, trigger.owner, ref: params[:ref], variables_attributes: variables) .execute(:trigger, ignore_skip_ci: true) do |pipeline| pipeline.trigger_requests.build(trigger: trigger) - pipeline.variables.build(variables) end if pipeline.persisted? @@ -44,7 +44,8 @@ module Ci # this check is to not leak the presence of the project if user cannot read it return unless can?(job.user, :read_project, project) - pipeline = Ci::CreatePipelineService.new(project, job.user, ref: params[:ref]) + pipeline = Ci::CreatePipelineService + .new(project, job.user, ref: params[:ref], variables_attributes: variables) .execute(:pipeline, ignore_skip_ci: true) do |pipeline| source = job.sourced_pipelines.build( source_pipeline: job.pipeline, @@ -53,7 +54,6 @@ module Ci project: project) pipeline.source_pipeline = source - pipeline.variables.build(variables) end if pipeline.persisted? diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb index 6adeca624a8..ebc980a9053 100644 --- a/app/services/ci/play_build_service.rb +++ b/app/services/ci/play_build_service.rb @@ -5,6 +5,10 @@ module Ci def execute(build, job_variables_attributes = nil) raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build) + if job_variables_attributes.present? && !can?(current_user, :set_pipeline_variables, project) + raise Gitlab::Access::AccessDeniedError + end + # Try to enqueue the build, otherwise create a duplicate. # if build.enqueue diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 12cdca24066..dd7b562cdb7 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -26,6 +26,27 @@ module Ci end def valid_statuses_for_build(build) + if ::Feature.enabled?(:skip_dag_manual_and_delayed_jobs, default_enabled: :yaml) + current_valid_statuses_for_build(build) + else + legacy_valid_statuses_for_build(build) + end + end + + def current_valid_statuses_for_build(build) + case build.when + when 'on_success', 'manual', 'delayed' + build.scheduling_type_dag? ? %w[success] : %w[success skipped] + when 'on_failure' + %w[failed] + when 'always' + %w[success failed skipped] + else + [] + end + end + + def legacy_valid_statuses_for_build(build) case build.when when 'on_success' build.scheduling_type_dag? ? %w[success] : %w[success skipped] diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 04d620d1d38..59691fe4ef3 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -8,8 +8,8 @@ module Ci JOB_QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze - METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'.freeze - DEFAULT_METRICS_SHARD = 'default'.freeze + METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' + DEFAULT_METRICS_SHARD = 'default' Result = Struct.new(:build, :build_json, :valid?) diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index f397ada0696..e5e79f70616 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -2,6 +2,8 @@ module Ci class RetryBuildService < ::BaseService + include Gitlab::OptimisticLocking + def self.clone_accessors %i[pipeline project ref tag options name allow_failure stage stage_id stage_idx trigger_request @@ -65,8 +67,8 @@ module Ci end def mark_subsequent_stages_as_processable(build) - build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |processable| - Gitlab::OptimisticLocking.retry_lock(processable, &:process) + build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |skipped| + retry_optimistic_lock(skipped) { |build| build.process(current_user) } end end end diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb index 45244d16393..dea4bf73a4c 100644 --- a/app/services/ci/retry_pipeline_service.rb +++ b/app/services/ci/retry_pipeline_service.rb @@ -23,7 +23,7 @@ module Ci end pipeline.builds.latest.skipped.find_each do |skipped| - retry_optimistic_lock(skipped) { |build| build.process } + retry_optimistic_lock(skipped) { |build| build.process(current_user) } end pipeline.reset_ancestor_bridges! diff --git a/app/services/ci/test_failure_history_service.rb b/app/services/ci/test_failure_history_service.rb index 99a2592ec06..61fda79a4a2 100644 --- a/app/services/ci/test_failure_history_service.rb +++ b/app/services/ci/test_failure_history_service.rb @@ -30,7 +30,6 @@ module Ci end def should_track_failures? - return false unless Feature.enabled?(:test_failure_history, project) return false unless project.default_branch_or_master == pipeline.ref # We fetch for up to MAX_TRACKABLE_FAILURES + 1 builds. So if ever we get diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb index f01d41d9414..874f4bf459a 100644 --- a/app/services/ci/update_build_state_service.rb +++ b/app/services/ci/update_build_state_service.rb @@ -111,7 +111,7 @@ module Ci Result.new(status: 200) when 'failed' - build.drop!(params[:failure_reason] || :unknown_failure) + build.drop_with_exit_code!(params[:failure_reason] || :unknown_failure, params[:exit_code]) Result.new(status: 200) else diff --git a/app/services/concerns/schedule_bulk_repository_shard_moves_methods.rb b/app/services/concerns/schedule_bulk_repository_shard_moves_methods.rb new file mode 100644 index 00000000000..eb03f3a9f3a --- /dev/null +++ b/app/services/concerns/schedule_bulk_repository_shard_moves_methods.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module ScheduleBulkRepositoryShardMovesMethods + extend ActiveSupport::Concern + include BaseServiceUtility + + class_methods do + def enqueue(source_storage_name, destination_storage_name = nil) + schedule_bulk_worker_klass.perform_async(source_storage_name, destination_storage_name) + end + + def schedule_bulk_worker_klass + raise NotImplementedError + end + end + + def execute(source_storage_name, destination_storage_name = nil) + shard = Shard.find_by_name!(source_storage_name) + + repository_klass.for_shard(shard).each_batch(column: container_column) do |relation| + container_klass.id_in(relation.select(container_column)).each do |container| + container.with_lock do + next if container.repository_storage != source_storage_name + + storage_move = container.repository_storage_moves.build( + source_storage_name: source_storage_name, + destination_storage_name: destination_storage_name + ) + + unless storage_move.schedule + log_info("Container #{container.full_path} (#{container.id}) was skipped: #{storage_move.errors.full_messages.to_sentence}") + end + end + end + end + + success + end + + private + + def repository_klass + raise NotImplementedError + end + + def container_klass + raise NotImplementedError + end + + def container_column + raise NotImplementedError + end +end diff --git a/app/services/concerns/update_repository_storage_methods.rb b/app/services/concerns/update_repository_storage_methods.rb new file mode 100644 index 00000000000..c3a55e9379e --- /dev/null +++ b/app/services/concerns/update_repository_storage_methods.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module UpdateRepositoryStorageMethods + Error = Class.new(StandardError) + SameFilesystemError = Class.new(Error) + + attr_reader :repository_storage_move + delegate :container, :source_storage_name, :destination_storage_name, to: :repository_storage_move + + def initialize(repository_storage_move) + @repository_storage_move = repository_storage_move + end + + def execute + repository_storage_move.with_lock do + return ServiceResponse.success unless repository_storage_move.scheduled? # rubocop:disable Cop/AvoidReturnFromBlocks + + repository_storage_move.start! + end + + raise SameFilesystemError if same_filesystem?(source_storage_name, destination_storage_name) + + mirror_repositories + + repository_storage_move.transaction do + repository_storage_move.finish_replication! + + track_repository(destination_storage_name) + end + + remove_old_paths + enqueue_housekeeping + + repository_storage_move.finish_cleanup! + + ServiceResponse.success + rescue StandardError => e + repository_storage_move.do_fail! + + Gitlab::ErrorTracking.track_exception(e, container_klass: container.class.to_s, container_path: container.full_path) + + ServiceResponse.error( + message: s_("UpdateRepositoryStorage|Error moving repository storage for %{container_full_path} - %{message}") % { container_full_path: container.full_path, message: e.message } + ) + end + + private + + def track_repository(destination_shard) + raise NotImplementedError + end + + def mirror_repositories + raise NotImplementedError + end + + def mirror_repository(type:) + unless wait_for_pushes(type) + raise Error, s_('UpdateRepositoryStorage|Timeout waiting for %{type} repository pushes') % { type: type.name } + end + + repository = type.repository_for(container) + full_path = repository.full_path + raw_repository = repository.raw + checksum = repository.checksum + + # Initialize a git repository on the target path + new_repository = Gitlab::Git::Repository.new( + destination_storage_name, + raw_repository.relative_path, + raw_repository.gl_repository, + full_path + ) + + new_repository.replicate(raw_repository) + new_checksum = new_repository.checksum + + if checksum != new_checksum + raise Error, s_('UpdateRepositoryStorage|Failed to verify %{type} repository checksum from %{old} to %{new}') % { type: type.name, old: checksum, new: new_checksum } + end + end + + def same_filesystem?(old_storage, new_storage) + Gitlab::GitalyClient.filesystem_id(old_storage) == Gitlab::GitalyClient.filesystem_id(new_storage) + end + + def remove_old_paths + if container.repository_exists? + Gitlab::Git::Repository.new( + source_storage_name, + "#{container.disk_path}.git", + nil, + nil + ).remove + end + end + + def enqueue_housekeeping + # no-op + end + + def wait_for_pushes(type) + reference_counter = container.reference_counter(type: type) + + # Try for 30 seconds, polling every 10 + 3.times do + return true if reference_counter.value == 0 + + sleep 10 + end + + false + end +end diff --git a/app/services/container_expiration_policies/cleanup_service.rb b/app/services/container_expiration_policies/cleanup_service.rb index 4719c99af6d..b9e623e2e07 100644 --- a/app/services/container_expiration_policies/cleanup_service.rb +++ b/app/services/container_expiration_policies/cleanup_service.rb @@ -4,6 +4,8 @@ module ContainerExpirationPolicies class CleanupService attr_reader :repository + SERVICE_RESULT_FIELDS = %i[original_size before_truncate_size after_truncate_size before_delete_size].freeze + def initialize(repository) @repository = repository end @@ -13,28 +15,37 @@ module ContainerExpirationPolicies repository.start_expiration_policy! - result = Projects::ContainerRepository::CleanupTagsService + service_result = Projects::ContainerRepository::CleanupTagsService .new(project, nil, policy_params.merge('container_expiration_policy' => true)) .execute(repository) - if result[:status] == :success + if service_result[:status] == :success repository.update!( expiration_policy_cleanup_status: :cleanup_unscheduled, expiration_policy_started_at: nil, expiration_policy_completed_at: Time.zone.now ) - success(:finished) + success(:finished, service_result) else repository.cleanup_unfinished! - success(:unfinished) + success(:unfinished, service_result) end end private - def success(cleanup_status) - ServiceResponse.success(message: "cleanup #{cleanup_status}", payload: { cleanup_status: cleanup_status, container_repository_id: repository.id }) + def success(cleanup_status, service_result) + payload = { + cleanup_status: cleanup_status, + container_repository_id: repository.id + } + + SERVICE_RESULT_FIELDS.each do |field| + payload["cleanup_tags_service_#{field}".to_sym] = service_result[field] + end + + ServiceResponse.success(message: "cleanup #{cleanup_status}", payload: payload) end def policy_params diff --git a/app/services/draft_notes/base_service.rb b/app/services/draft_notes/base_service.rb index 89daae0e8f4..95c291ea800 100644 --- a/app/services/draft_notes/base_service.rb +++ b/app/services/draft_notes/base_service.rb @@ -8,6 +8,10 @@ module DraftNotes @merge_request, @current_user, @params = merge_request, current_user, params.dup end + def merge_request_activity_counter + Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter + end + private def draft_notes diff --git a/app/services/draft_notes/create_service.rb b/app/services/draft_notes/create_service.rb index 501778b7d5f..5ff971b66c1 100644 --- a/app/services/draft_notes/create_service.rb +++ b/app/services/draft_notes/create_service.rb @@ -31,6 +31,10 @@ module DraftNotes merge_request.diffs.clear_cache end + if draft_note.persisted? + merge_request_activity_counter.track_create_review_note_action(user: current_user) + end + draft_note end diff --git a/app/services/draft_notes/publish_service.rb b/app/services/draft_notes/publish_service.rb index a9a7304e5ed..316abff4552 100644 --- a/app/services/draft_notes/publish_service.rb +++ b/app/services/draft_notes/publish_service.rb @@ -9,6 +9,7 @@ module DraftNotes publish_draft_note(draft) else publish_draft_notes + merge_request_activity_counter.track_publish_review_action(user: current_user) end success diff --git a/app/services/feature_flags/base_service.rb b/app/services/feature_flags/base_service.rb index 9b27df90992..c11c465252e 100644 --- a/app/services/feature_flags/base_service.rb +++ b/app/services/feature_flags/base_service.rb @@ -6,6 +6,11 @@ module FeatureFlags AUDITABLE_ATTRIBUTES = %w(name description active).freeze + def success(**args) + sync_to_jira(args[:feature_flag]) + super + end + protected def audit_event(feature_flag) @@ -34,6 +39,16 @@ module FeatureFlags audit_event.security_event end + def sync_to_jira(feature_flag) + return unless feature_flag.present? + return unless Feature.enabled?(:jira_sync_feature_flags, feature_flag.project) + + seq_id = ::Atlassian::JiraConnect::Client.generate_update_sequence_id + feature_flag.run_after_commit do + ::JiraConnect::SyncFeatureFlagsWorker.perform_async(feature_flag.id, seq_id) + end + end + def created_scope_message(scope) "Created rule <strong>#{scope.environment_scope}</strong> "\ "and set it as <strong>#{scope.active ? "active" : "inactive"}</strong> "\ diff --git a/app/services/git/branch_push_service.rb b/app/services/git/branch_push_service.rb index 2ec6ac99ece..d250bca7bf2 100644 --- a/app/services/git/branch_push_service.rb +++ b/app/services/git/branch_push_service.rb @@ -72,10 +72,10 @@ module Git end def perform_housekeeping - housekeeping = Projects::HousekeepingService.new(project) + housekeeping = Repositories::HousekeepingService.new(project) housekeeping.increment! housekeeping.execute if housekeeping.needed? - rescue Projects::HousekeepingService::LeaseTaken + rescue Repositories::HousekeepingService::LeaseTaken end def removing_branch? diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb index 52600f5b88f..06a3b31c665 100644 --- a/app/services/groups/create_service.rb +++ b/app/services/groups/create_service.rb @@ -35,6 +35,7 @@ module Groups @group.add_owner(current_user) @group.create_namespace_settings Service.create_from_active_default_integrations(@group, :group_id) + OnboardingProgress.onboard(@group) end end diff --git a/app/services/groups/destroy_service.rb b/app/services/groups/destroy_service.rb index 1bff70e6c2e..c7107e2fa56 100644 --- a/app/services/groups/destroy_service.rb +++ b/app/services/groups/destroy_service.rb @@ -29,17 +29,32 @@ module Groups group.chat_team&.remove_mattermost_team(current_user) - user_ids_for_project_authorizations_refresh = group.user_ids_for_project_authorizations + # If any other groups are shared with the group that is being destroyed, + # we should specifically trigger update of all project authorizations + # for users that are the members of this group. + # If not, the project authorization records of these users to projects within the shared groups + # will never be removed, causing inconsistencies with access permissions. + if any_other_groups_are_shared_with_this_group? + user_ids_for_project_authorizations_refresh = group.user_ids_for_project_authorizations + end group.destroy - UserProjectAccessChangedService - .new(user_ids_for_project_authorizations_refresh) - .execute(blocking: true) + if user_ids_for_project_authorizations_refresh.present? + UserProjectAccessChangedService + .new(user_ids_for_project_authorizations_refresh) + .execute(blocking: true) + end group end # rubocop: enable CodeReuse/ActiveRecord + + private + + def any_other_groups_are_shared_with_this_group? + group.shared_group_links.any? + end end end diff --git a/app/services/ide/base_config_service.rb b/app/services/ide/base_config_service.rb index 1f8d5c17584..0501fab53af 100644 --- a/app/services/ide/base_config_service.rb +++ b/app/services/ide/base_config_service.rb @@ -4,7 +4,7 @@ module Ide class BaseConfigService < ::BaseService ValidationError = Class.new(StandardError) - WEBIDE_CONFIG_FILE = '.gitlab/.gitlab-webide.yml'.freeze + WEBIDE_CONFIG_FILE = '.gitlab/.gitlab-webide.yml' attr_reader :config, :config_content diff --git a/app/services/incident_management/pager_duty/process_webhook_service.rb b/app/services/incident_management/pager_duty/process_webhook_service.rb index 027425e4aaa..ccbca671b37 100644 --- a/app/services/incident_management/pager_duty/process_webhook_service.rb +++ b/app/services/incident_management/pager_duty/process_webhook_service.rb @@ -2,7 +2,7 @@ module IncidentManagement module PagerDuty - class ProcessWebhookService < BaseService + class ProcessWebhookService include Gitlab::Utils::StrongMemoize include IncidentManagement::Settings @@ -12,6 +12,11 @@ module IncidentManagement # https://developer.pagerduty.com/docs/webhooks/v2-overview/#webhook-types PAGER_DUTY_PROCESSABLE_EVENT_TYPES = %w(incident.trigger).freeze + def initialize(project, payload) + @project = project + @payload = payload + end + def execute(token) return forbidden unless webhook_setting_active? return unauthorized unless valid_token?(token) @@ -24,6 +29,8 @@ module IncidentManagement private + attr_reader :project, :payload + def process_incidents pager_duty_processable_events.each do |event| ::IncidentManagement::PagerDuty::ProcessIncidentWorker.perform_async(project.id, event['incident']) @@ -33,7 +40,7 @@ module IncidentManagement def pager_duty_processable_events strong_memoize(:pager_duty_processable_events) do ::PagerDuty::WebhookPayloadParser - .call(params.to_h) + .call(payload.to_h) .filter { |msg| msg['event'].to_s.in?(PAGER_DUTY_PROCESSABLE_EVENT_TYPES) } end end @@ -47,7 +54,7 @@ module IncidentManagement end def valid_payload_size? - Gitlab::Utils::DeepSize.new(params, max_size: PAGER_DUTY_PAYLOAD_SIZE_LIMIT).valid? + Gitlab::Utils::DeepSize.new(payload, max_size: PAGER_DUTY_PAYLOAD_SIZE_LIMIT).valid? end def accepted diff --git a/app/services/issuable/bulk_update_service.rb b/app/services/issuable/bulk_update_service.rb index 79be771b3fb..d3d543edcd7 100644 --- a/app/services/issuable/bulk_update_service.rb +++ b/app/services/issuable/bulk_update_service.rb @@ -34,6 +34,8 @@ module Issuable def permitted_attrs(type) attrs = %i(state_event milestone_id add_label_ids remove_label_ids subscription_event) + attrs.push(:sprint_id) if type == 'issue' + if type == 'issue' || type == 'merge_request' attrs.push(:assignee_ids) else diff --git a/app/services/issuable/export_csv/base_service.rb b/app/services/issuable/export_csv/base_service.rb new file mode 100644 index 00000000000..49ff05935c9 --- /dev/null +++ b/app/services/issuable/export_csv/base_service.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Issuable + module ExportCsv + class BaseService + # Target attachment size before base64 encoding + TARGET_FILESIZE = 15.megabytes + + def initialize(issuables_relation, project) + @issuables = issuables_relation + @project = project + end + + def csv_data + csv_builder.render(TARGET_FILESIZE) + end + + private + + attr_reader :project, :issuables + + # rubocop: disable CodeReuse/ActiveRecord + def csv_builder + @csv_builder ||= + CsvBuilder.new(issuables.preload(associations_to_preload), header_to_value_hash) + end + # rubocop: enable CodeReuse/ActiveRecord + + def associations_to_preload + [] + end + + def header_to_value_hash + raise NotImplementedError + end + end + end +end diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 60e5293e218..6d41d449683 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -158,7 +158,9 @@ class IssuableBaseService < BaseService after_create(issuable) execute_hooks(issuable) - invalidate_cache_counts(issuable, users: issuable.assignees) + + users_to_invalidate = issuable.allows_reviewers? ? issuable.assignees | issuable.reviewers : issuable.assignees + invalidate_cache_counts(issuable, users: users_to_invalidate) issuable.update_project_counter_caches end diff --git a/app/services/issues/clone_service.rb b/app/services/issues/clone_service.rb index 789da312958..4c9c34f1247 100644 --- a/app/services/issues/clone_service.rb +++ b/app/services/issues/clone_service.rb @@ -88,3 +88,5 @@ module Issues end end end + +Issues::CloneService.prepend_if_ee('EE::Issues::CloneService') diff --git a/app/services/issues/close_service.rb b/app/services/issues/close_service.rb index c3677de015f..baf7974c45d 100644 --- a/app/services/issues/close_service.rb +++ b/app/services/issues/close_service.rb @@ -31,7 +31,7 @@ module Issues closed_via = _("commit %{commit_id}") % { commit_id: closed_via.id } if closed_via.is_a?(Commit) - notification_service.async.close_issue(issue, current_user, closed_via: closed_via) if notifications + notification_service.async.close_issue(issue, current_user, { closed_via: closed_via }) if notifications todo_service.close_issue(issue, current_user) resolve_alert(issue) execute_hooks(issue, 'close') @@ -39,7 +39,10 @@ module Issues issue.update_project_counter_caches track_incident_action(current_user, issue, :incident_closed) - store_first_mentioned_in_commit_at(issue, closed_via) if closed_via.is_a?(MergeRequest) + if closed_via.is_a?(MergeRequest) + store_first_mentioned_in_commit_at(issue, closed_via) + OnboardingProgressService.new(project.namespace).execute(action: :issue_auto_closed) + end delete_milestone_closed_issue_counter_cache(issue.milestone) end diff --git a/app/services/issues/export_csv_service.rb b/app/services/issues/export_csv_service.rb index 8f513632929..2181c46c90d 100644 --- a/app/services/issues/export_csv_service.rb +++ b/app/services/issues/export_csv_service.rb @@ -1,36 +1,14 @@ # frozen_string_literal: true module Issues - class ExportCsvService + class ExportCsvService < Issuable::ExportCsv::BaseService include Gitlab::Routing.url_helpers include GitlabRoutingHelper - # Target attachment size before base64 encoding - TARGET_FILESIZE = 15000000 - - attr_reader :project - - def initialize(issues_relation, project) - @issues = issues_relation - @labels = @issues.labels_hash - @project = project - end - - def csv_data - csv_builder.render(TARGET_FILESIZE) - end - def email(user) Notify.issues_csv_email(user, project, csv_data, csv_builder.status).deliver_now end - # rubocop: disable CodeReuse/ActiveRecord - def csv_builder - @csv_builder ||= - CsvBuilder.new(@issues.preload(associations_to_preload), header_to_value_hash) - end - # rubocop: enable CodeReuse/ActiveRecord - private def associations_to_preload @@ -63,7 +41,7 @@ module Issues end def issue_labels(issue) - @labels[issue.id].sort.join(',').presence + issuables.labels_hash[issue.id].sort.join(',').presence end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/services/jira_connect/sync_service.rb b/app/services/jira_connect/sync_service.rb index b2af284f1f0..bddc7cbe5a0 100644 --- a/app/services/jira_connect/sync_service.rb +++ b/app/services/jira_connect/sync_service.rb @@ -31,7 +31,7 @@ module JiraConnect jira_response: response&.to_json } - if response && (response['errorMessages'] || response['rejectedBuilds'].present?) + if response && response['errorMessages'].present? logger.error(message) else logger.info(message) diff --git a/app/services/jira_connect_subscriptions/create_service.rb b/app/services/jira_connect_subscriptions/create_service.rb index b169d97615d..38e5fe7e690 100644 --- a/app/services/jira_connect_subscriptions/create_service.rb +++ b/app/services/jira_connect_subscriptions/create_service.rb @@ -35,8 +35,6 @@ module JiraConnectSubscriptions end def schedule_sync_project_jobs - return unless Feature.enabled?(:jira_connect_full_namespace_sync) - namespace.all_projects.each_batch(of: MERGE_REQUEST_SYNC_BATCH_SIZE) do |projects, index| JiraConnect::SyncProjectWorker.bulk_perform_in_with_contexts( index * MERGE_REQUEST_SYNC_BATCH_DELAY, diff --git a/app/services/labels/create_service.rb b/app/services/labels/create_service.rb index a5b30e29e55..665d1035b2b 100644 --- a/app/services/labels/create_service.rb +++ b/app/services/labels/create_service.rb @@ -25,3 +25,5 @@ module Labels end end end + +Labels::CreateService.prepend_if_ee('EE::Labels::CreateService') diff --git a/app/services/members/create_service.rb b/app/services/members/create_service.rb index 3588cda180f..5fcf2d711b0 100644 --- a/app/services/members/create_service.rb +++ b/app/services/members/create_service.rb @@ -54,7 +54,8 @@ module Members end def enqueue_onboarding_progress_action(source) - Namespaces::OnboardingUserAddedWorker.perform_async(source.id) + namespace_id = source.is_a?(Project) ? source.namespace_id : source.id + Namespaces::OnboardingUserAddedWorker.perform_async(namespace_id) end end end diff --git a/app/services/merge_requests/after_create_service.rb b/app/services/merge_requests/after_create_service.rb index fbb9d5fa9dc..03fcb5a4c1b 100644 --- a/app/services/merge_requests/after_create_service.rb +++ b/app/services/merge_requests/after_create_service.rb @@ -4,6 +4,7 @@ module MergeRequests class AfterCreateService < MergeRequests::BaseService def execute(merge_request) event_service.open_mr(merge_request, current_user) + merge_request_activity_counter.track_create_mr_action(user: current_user) notification_service.new_merge_request(merge_request, current_user) create_pipeline_for(merge_request, current_user) @@ -12,7 +13,7 @@ module MergeRequests merge_request.diffs(include_stats: false).write_cache merge_request.create_cross_references!(current_user) - NamespaceOnboardingAction.create_action(merge_request.target_project.namespace, :merge_request_created) + OnboardingProgressService.new(merge_request.target_project.namespace).execute(action: :merge_request_created) end end end diff --git a/app/services/merge_requests/base_service.rb b/app/services/merge_requests/base_service.rb index 265b211066e..0613c061f2e 100644 --- a/app/services/merge_requests/base_service.rb +++ b/app/services/merge_requests/base_service.rb @@ -52,6 +52,10 @@ module MergeRequests "#<#{self.class} #{merge_request.to_reference(full: true)}>" end + def merge_request_activity_counter + Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter + end + private def enqueue_jira_connect_messages_for(merge_request) diff --git a/app/services/merge_requests/cleanup_refs_service.rb b/app/services/merge_requests/cleanup_refs_service.rb index 23ac8e393f4..2094ea00160 100644 --- a/app/services/merge_requests/cleanup_refs_service.rb +++ b/app/services/merge_requests/cleanup_refs_service.rb @@ -36,6 +36,8 @@ module MergeRequests return error('Failed to update schedule.') unless update_schedule success + rescue Gitlab::Git::Repository::GitError, Gitlab::Git::CommandError => e + error(e.message) end private diff --git a/app/services/merge_requests/close_service.rb b/app/services/merge_requests/close_service.rb index b0a7face594..f83b14c7269 100644 --- a/app/services/merge_requests/close_service.rb +++ b/app/services/merge_requests/close_service.rb @@ -13,11 +13,12 @@ module MergeRequests if merge_request.close create_event(merge_request) + merge_request_activity_counter.track_close_mr_action(user: current_user) create_note(merge_request) notification_service.async.close_mr(merge_request, current_user) todo_service.close_merge_request(merge_request, current_user) execute_hooks(merge_request, 'close') - invalidate_cache_counts(merge_request, users: merge_request.assignees) + invalidate_cache_counts(merge_request, users: merge_request.assignees | merge_request.reviewers) merge_request.update_project_counter_caches cleanup_environments(merge_request) abort_auto_merge(merge_request, 'merge request was closed') diff --git a/app/services/merge_requests/create_from_issue_service.rb b/app/services/merge_requests/create_from_issue_service.rb index 95fb99d3e7a..78b462174c9 100644 --- a/app/services/merge_requests/create_from_issue_service.rb +++ b/app/services/merge_requests/create_from_issue_service.rb @@ -85,7 +85,8 @@ module MergeRequests source_project_id: target_project.id, source_branch: branch_name, target_project_id: target_project.id, - target_branch: target_branch + target_branch: target_branch, + assignee_ids: [current_user.id] } end diff --git a/app/services/merge_requests/export_csv_service.rb b/app/services/merge_requests/export_csv_service.rb index 1e7f0c8e722..8f2a70575e5 100644 --- a/app/services/merge_requests/export_csv_service.rb +++ b/app/services/merge_requests/export_csv_service.rb @@ -1,32 +1,16 @@ # frozen_string_literal: true module MergeRequests - class ExportCsvService + class ExportCsvService < Issuable::ExportCsv::BaseService include Gitlab::Routing.url_helpers include GitlabRoutingHelper - # Target attachment size before base64 encoding - TARGET_FILESIZE = 15.megabytes - - def initialize(merge_requests, project) - @project = project - @merge_requests = merge_requests - end - - def csv_data - csv_builder.render(TARGET_FILESIZE) - end - def email(user) - Notify.merge_requests_csv_email(user, @project, csv_data, csv_builder.status).deliver_now + Notify.merge_requests_csv_email(user, project, csv_data, csv_builder.status).deliver_now end private - def csv_builder - @csv_builder ||= CsvBuilder.new(@merge_requests.with_csv_entity_associations, header_to_value_hash) - end - def header_to_value_hash { 'MR IID' => 'iid', diff --git a/app/services/merge_requests/merge_service.rb b/app/services/merge_requests/merge_service.rb index ba22b458777..f4454db0af8 100644 --- a/app/services/merge_requests/merge_service.rb +++ b/app/services/merge_requests/merge_service.rb @@ -88,7 +88,9 @@ module MergeRequests end def try_merge - repository.merge(current_user, source, merge_request, commit_message) + repository.merge(current_user, source, merge_request, commit_message).tap do + merge_request.update_column(:squash_commit_sha, source) if merge_request.squash_on_merge? + end rescue Gitlab::Git::PreReceiveError => e raise MergeError, "Something went wrong during merge pre-receive hook. #{e.message}".strip diff --git a/app/services/merge_requests/mergeability_check_service.rb b/app/services/merge_requests/mergeability_check_service.rb index 627c747203c..96a2322f6a0 100644 --- a/app/services/merge_requests/mergeability_check_service.rb +++ b/app/services/merge_requests/mergeability_check_service.rb @@ -38,7 +38,6 @@ module MergeRequests # error otherwise. def execute(recheck: false, retry_lease: true) return service_error if service_error - return check_mergeability(recheck) unless merge_ref_auto_sync_lock_enabled? in_write_lock(retry_lease: retry_lease) do |retried| # When multiple calls are waiting for the same lock (retry_lease), @@ -64,10 +63,6 @@ module MergeRequests return ServiceResponse.error(message: 'Merge request is not mergeable') end - unless merge_ref_auto_sync_enabled? - return ServiceResponse.error(message: 'Merge ref is outdated due to disabled feature') - end - unless payload.fetch(:merge_ref_head) return ServiceResponse.error(message: 'Merge ref cannot be updated') end @@ -142,7 +137,6 @@ module MergeRequests # # Returns true if the merge-ref does not exists or is out of sync. def outdated_merge_ref? - return false unless merge_ref_auto_sync_enabled? return false unless merge_request.open? return true unless ref_head = merge_request.merge_ref_head @@ -157,21 +151,11 @@ module MergeRequests end def merge_to_ref - return true unless merge_ref_auto_sync_enabled? - params = { allow_conflicts: Feature.enabled?(:display_merge_conflicts_in_diff, project) } result = MergeRequests::MergeToRefService.new(project, merge_request.author, params).execute(merge_request) result[:status] == :success end - def merge_ref_auto_sync_enabled? - Feature.enabled?(:merge_ref_auto_sync, project, default_enabled: true) - end - - def merge_ref_auto_sync_lock_enabled? - Feature.enabled?(:merge_ref_auto_sync_lock, project, default_enabled: true) - end - def service_error strong_memoize(:service_error) do if !merge_request diff --git a/app/services/merge_requests/post_merge_service.rb b/app/services/merge_requests/post_merge_service.rb index 1c78fca3c26..f04ec3c3e80 100644 --- a/app/services/merge_requests/post_merge_service.rb +++ b/app/services/merge_requests/post_merge_service.rb @@ -15,9 +15,10 @@ module MergeRequests todo_service.merge_merge_request(merge_request, current_user) create_event(merge_request) create_note(merge_request) + merge_request_activity_counter.track_merge_mr_action(user: current_user) notification_service.merge_mr(merge_request, current_user) execute_hooks(merge_request, 'merge') - invalidate_cache_counts(merge_request, users: merge_request.assignees) + invalidate_cache_counts(merge_request, users: merge_request.assignees | merge_request.reviewers) merge_request.update_project_counter_caches delete_non_latest_diffs(merge_request) cancel_review_app_jobs!(merge_request) diff --git a/app/services/merge_requests/reopen_service.rb b/app/services/merge_requests/reopen_service.rb index bcedbc61c65..35c50d63da0 100644 --- a/app/services/merge_requests/reopen_service.rb +++ b/app/services/merge_requests/reopen_service.rb @@ -8,11 +8,12 @@ module MergeRequests if merge_request.reopen create_event(merge_request) create_note(merge_request, 'reopened') + merge_request_activity_counter.track_reopen_mr_action(user: current_user) notification_service.async.reopen_mr(merge_request, current_user) execute_hooks(merge_request, 'reopen') merge_request.reload_diff(current_user) merge_request.mark_as_unchecked - invalidate_cache_counts(merge_request, users: merge_request.assignees) + invalidate_cache_counts(merge_request, users: merge_request.assignees | merge_request.reviewers) merge_request.update_project_counter_caches merge_request.cache_merge_request_closes_issues!(current_user) merge_request.cleanup_schedule&.destroy diff --git a/app/services/merge_requests/update_service.rb b/app/services/merge_requests/update_service.rb index bff7a43dd7b..d2e5a2a1619 100644 --- a/app/services/merge_requests/update_service.rb +++ b/app/services/merge_requests/update_service.rb @@ -112,9 +112,11 @@ module MergeRequests end def handle_reviewers_change(merge_request, old_reviewers) + affected_reviewers = (old_reviewers + merge_request.reviewers) - (old_reviewers & merge_request.reviewers) create_reviewer_note(merge_request, old_reviewers) notification_service.async.changed_reviewer_of_merge_request(merge_request, current_user, old_reviewers) todo_service.reassigned_reviewable(merge_request, current_user, old_reviewers) + invalidate_cache_counts(merge_request, users: affected_reviewers.compact) end def create_branch_change_note(issuable, branch_type, old_branch, new_branch) @@ -126,27 +128,29 @@ module MergeRequests override :handle_quick_actions def handle_quick_actions(merge_request) super + + # Ensure this parameter does not get used as an attribute + rebase = params.delete(:rebase) + + if rebase + rebase_from_quick_action(merge_request) + # Ignore "/merge" if "/rebase" is used to avoid an unexpected race + params.delete(:merge) + end + merge_from_quick_action(merge_request) if params[:merge] end + def rebase_from_quick_action(merge_request) + merge_request.rebase_async(current_user.id) + end + def merge_from_quick_action(merge_request) last_diff_sha = params.delete(:merge) - if Feature.enabled?(:merge_orchestration_service, merge_request.project, default_enabled: true) - MergeRequests::MergeOrchestrationService - .new(project, current_user, { sha: last_diff_sha }) - .execute(merge_request) - else - return unless merge_request.mergeable_with_quick_action?(current_user, last_diff_sha: last_diff_sha) - - merge_request.update(merge_error: nil) - - if merge_request.head_pipeline_active? - AutoMergeService.new(project, current_user, { sha: last_diff_sha }).execute(merge_request, AutoMergeService::STRATEGY_MERGE_WHEN_PIPELINE_SUCCEEDS) - else - merge_request.merge_async(current_user.id, { sha: last_diff_sha }) - end - end + MergeRequests::MergeOrchestrationService + .new(project, current_user, { sha: last_diff_sha }) + .execute(merge_request) end override :quick_action_options diff --git a/app/services/namespaces/package_settings/update_service.rb b/app/services/namespaces/package_settings/update_service.rb new file mode 100644 index 00000000000..0964963647a --- /dev/null +++ b/app/services/namespaces/package_settings/update_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Namespaces + module PackageSettings + class UpdateService < BaseContainerService + include Gitlab::Utils::StrongMemoize + + ALLOWED_ATTRIBUTES = %i[maven_duplicates_allowed maven_duplicate_exception_regex].freeze + + def execute + return ServiceResponse.error(message: 'Access Denied', http_status: 403) unless allowed? + + if package_settings.update(package_settings_params) + ServiceResponse.success(payload: { package_settings: package_settings }) + else + ServiceResponse.error( + message: package_settings.errors.full_messages.to_sentence || 'Bad request', + http_status: 400 + ) + end + end + + private + + def package_settings + strong_memoize(:package_settings) do + @container.package_settings + end + end + + def allowed? + Ability.allowed?(current_user, :create_package_settings, @container) + end + + def package_settings_params + @params.slice(*ALLOWED_ATTRIBUTES) + end + end + end +end diff --git a/app/services/notes/create_service.rb b/app/services/notes/create_service.rb index 9fffb6c372b..04b7fba207b 100644 --- a/app/services/notes/create_service.rb +++ b/app/services/notes/create_service.rb @@ -75,6 +75,7 @@ module Notes end track_note_creation_usage_for_issues(note) if note.for_issue? + track_note_creation_usage_for_merge_requests(note) if note.for_merge_request? end def do_commands(note, update_params, message, only_commands) @@ -119,5 +120,9 @@ module Notes def track_note_creation_usage_for_issues(note) Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_comment_added_action(author: note.author) end + + def track_note_creation_usage_for_merge_requests(note) + Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter.track_create_comment_action(note: note) + end end end diff --git a/app/services/notes/destroy_service.rb b/app/services/notes/destroy_service.rb index 2b6ec47eaef..85f54a39add 100644 --- a/app/services/notes/destroy_service.rb +++ b/app/services/notes/destroy_service.rb @@ -9,6 +9,7 @@ module Notes clear_noteable_diffs_cache(note) track_note_removal_usage_for_issues(note) if note.for_issue? + track_note_removal_usage_for_merge_requests(note) if note.for_merge_request? end private @@ -16,6 +17,10 @@ module Notes def track_note_removal_usage_for_issues(note) Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_comment_removed_action(author: note.author) end + + def track_note_removal_usage_for_merge_requests(note) + Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter.track_remove_comment_action(note: note) + end end end diff --git a/app/services/notes/update_service.rb b/app/services/notes/update_service.rb index 37872f7fbdb..857ffbb6965 100644 --- a/app/services/notes/update_service.rb +++ b/app/services/notes/update_service.rb @@ -15,6 +15,7 @@ module Notes end track_note_edit_usage_for_issues(note) if note.for_issue? + track_note_edit_usage_for_merge_requests(note) if note.for_merge_request? only_commands = false @@ -95,6 +96,10 @@ module Notes def track_note_edit_usage_for_issues(note) Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_comment_edited_action(author: note.author) end + + def track_note_edit_usage_for_merge_requests(note) + Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter.track_edit_comment_action(note: note) + end end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 4ff462191fe..5a71e0eac7c 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -118,8 +118,8 @@ class NotificationService # * project team members with notification level higher then Participating # * users with custom level checked with "close issue" # - def close_issue(issue, current_user, closed_via: nil) - close_resource_email(issue, current_user, :closed_issue_email, closed_via: closed_via) + def close_issue(issue, current_user, params = {}) + close_resource_email(issue, current_user, :closed_issue_email, closed_via: params[:closed_via]) end # When we reassign an issue we should send an email to: @@ -481,6 +481,12 @@ class NotificationService mailer.member_access_granted_email(group_member.real_source_type, group_member.id).deliver_later end + def updated_group_member_expiration(group_member) + return true unless group_member.notifiable?(:mention) + + mailer.member_expiration_date_updated_email(group_member.real_source_type, group_member.id).deliver_later + end + def project_was_moved(project, old_path_with_namespace) recipients = project_moved_recipients(project) recipients = notifiable_users(recipients, :custom, custom_action: :moved_project, project: project) diff --git a/app/services/onboarding_progress_service.rb b/app/services/onboarding_progress_service.rb index ebe7caabdef..241bd8a01ca 100644 --- a/app/services/onboarding_progress_service.rb +++ b/app/services/onboarding_progress_service.rb @@ -2,10 +2,12 @@ class OnboardingProgressService def initialize(namespace) - @namespace = namespace.root_ancestor + @namespace = namespace&.root_ancestor end def execute(action:) - NamespaceOnboardingAction.create_action(@namespace, action) + return unless @namespace + + OnboardingProgress.register(@namespace, action) end end diff --git a/app/services/packages/create_event_service.rb b/app/services/packages/create_event_service.rb index f0328ceb08a..63248ef07c9 100644 --- a/app/services/packages/create_event_service.rb +++ b/app/services/packages/create_event_service.rb @@ -3,11 +3,13 @@ module Packages class CreateEventService < BaseService def execute - if Feature.enabled?(:collect_package_events_redis) && redis_event_name - if guest? - ::Gitlab::UsageDataCounters::GuestPackageEventCounter.count(redis_event_name) - else - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(current_user.id, redis_event_name) + if Feature.enabled?(:collect_package_events_redis, default_enabled: true) + ::Packages::Event.unique_counters_for(event_scope, event_name, originator_type).each do |event_name| + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: current_user.id) + end + + ::Packages::Event.counters_for(event_scope, event_name, originator_type).each do |event_name| + ::Gitlab::UsageDataCounters::PackageEventCounter.count(event_name) end end @@ -23,10 +25,6 @@ module Packages private - def redis_event_name - @redis_event_name ||= ::Packages::Event.allowed_event_name(event_scope, event_name, originator_type) - end - def event_scope @event_scope ||= scope.is_a?(::Packages::Package) ? scope.package_type : scope end diff --git a/app/services/packages/debian/create_package_file_service.rb b/app/services/packages/debian/create_package_file_service.rb new file mode 100644 index 00000000000..2022a63a725 --- /dev/null +++ b/app/services/packages/debian/create_package_file_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Packages + module Debian + class CreatePackageFileService + def initialize(package, params) + @package = package + @params = params + end + + def execute + raise ArgumentError, "Invalid package" unless package.present? + + # Debian package file are first uploaded to incoming with empty metadata, + # and are moved later by Packages::Debian::ProcessChangesService + package.package_files.create!( + file: params[:file], + size: params[:file]&.size, + file_name: params[:file_name], + file_sha1: params[:file_sha1], + file_sha256: params[:file]&.sha256, + file_md5: params[:file_md5], + debian_file_metadatum_attributes: { + file_type: 'unknown', + architecture: nil, + fields: nil + } + ) + end + + private + + attr_reader :package, :params + end + end +end diff --git a/app/services/packages/debian/extract_metadata_service.rb b/app/services/packages/debian/extract_metadata_service.rb new file mode 100644 index 00000000000..fd5832bc0ba --- /dev/null +++ b/app/services/packages/debian/extract_metadata_service.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Packages + module Debian + class ExtractMetadataService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + def initialize(package_file) + @package_file = package_file + end + + def execute + raise ExtractionError.new('invalid package file') unless valid_package_file? + + extract_metadata + end + + private + + attr_reader :package_file + + def valid_package_file? + package_file && + package_file.package&.debian? && + package_file.file.size > 0 # rubocop:disable Style/ZeroLengthPredicate + end + + def file_type_basic + %i[dsc deb udeb buildinfo changes].each do |format| + return format if package_file.file_name.end_with?(".#{format}") + end + + nil + end + + def file_type_source + # https://manpages.debian.org/buster/dpkg-dev/dpkg-source.1.en.html + %i[gzip bzip2 lzma xz].each do |format| + return :source if package_file.file_name.end_with?(".tar.#{format}") + end + + nil + end + + def file_type + strong_memoize(:file_type) do + file_type_basic || file_type_source || :unknown + end + end + + def file_type_debian? + file_type == :deb || file_type == :udeb + end + + def file_type_meta? + file_type == :dsc || file_type == :buildinfo || file_type == :changes + end + + def extracted_fields + if file_type_debian? + package_file.file.use_file do |file_path| + ::Packages::Debian::ExtractDebMetadataService.new(file_path).execute + end + elsif file_type_meta? + package_file.file.use_file do |file_path| + ::Packages::Debian::ParseDebian822Service.new(File.read(file_path)).execute.each_value.first + end + end + end + + def extract_metadata + fields = extracted_fields + architecture = fields.delete(:Architecture) if file_type_debian? + + { + file_type: file_type, + architecture: architecture, + fields: fields + } + end + end + end +end diff --git a/app/services/packages/debian/get_or_create_incoming_service.rb b/app/services/packages/debian/get_or_create_incoming_service.rb new file mode 100644 index 00000000000..09e7877a2b4 --- /dev/null +++ b/app/services/packages/debian/get_or_create_incoming_service.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Packages + module Debian + class GetOrCreateIncomingService < ::Packages::CreatePackageService + def execute + find_or_create_package!(:debian, name: 'incoming', version: nil) + end + end + end +end diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb index f598b5e7cd4..8ee449cbfdc 100644 --- a/app/services/packages/maven/find_or_create_package_service.rb +++ b/app/services/packages/maven/find_or_create_package_service.rb @@ -2,14 +2,23 @@ module Packages module Maven class FindOrCreatePackageService < BaseService - MAVEN_METADATA_FILE = 'maven-metadata.xml'.freeze - SNAPSHOT_TERM = '-SNAPSHOT'.freeze + MAVEN_METADATA_FILE = 'maven-metadata.xml' + SNAPSHOT_TERM = '-SNAPSHOT' def execute package = ::Packages::Maven::PackageFinder.new(params[:path], current_user, project: project) .execute + unless Namespace::PackageSetting.duplicates_allowed?(package) + files = package&.package_files || [] + current_maven_files = files.map { |file| extname(file.file_name) } + + if current_maven_files.compact.include?(extname(params[:file_name])) + return ServiceResponse.error(message: 'Duplicate package is not allowed') + end + end + unless package # Maven uploads several files during `mvn deploy` in next order: # - my-company/my-app/1.0-SNAPSHOT/my-app.jar @@ -48,7 +57,15 @@ module Packages package.build_infos.safe_find_or_create_by!(pipeline: params[:build].pipeline) if params[:build].present? - package + ServiceResponse.success(payload: { package: package }) + end + + private + + def extname(filename) + return if filename.blank? + + File.extname(filename) end end end diff --git a/app/services/packages/nuget/search_service.rb b/app/services/packages/nuget/search_service.rb index b95aa30bec1..1eead1e62b3 100644 --- a/app/services/packages/nuget/search_service.rb +++ b/app/services/packages/nuget/search_service.rb @@ -3,6 +3,7 @@ module Packages module Nuget class SearchService < BaseService + include ::Packages::FinderHelper include Gitlab::Utils::StrongMemoize include ActiveRecord::ConnectionAdapters::Quoting @@ -16,8 +17,9 @@ module Packages padding: 0 }.freeze - def initialize(project, search_term, options = {}) - @project = project + def initialize(current_user, project_or_group, search_term, options = {}) + @current_user = current_user + @project_or_group = project_or_group @search_term = search_term @options = DEFAULT_OPTIONS.merge(options) @@ -26,8 +28,8 @@ module Packages end def execute - OpenStruct.new( - total_count: package_names.total_count, + Result.new( + total_count: non_paginated_matching_package_names.count, results: search_packages ) end @@ -39,52 +41,104 @@ module Packages # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24182#technical-notes # and https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource subquery_name = :partition_subquery - arel_table = Arel::Table.new(:partition_subquery) + arel_table = Arel::Table.new(subquery_name) column_names = Packages::Package.column_names.map do |cn| "#{subquery_name}.#{quote_column_name(cn)}" end # rubocop: disable CodeReuse/ActiveRecord - pkgs = Packages::Package.select(column_names.join(',')) - .from(package_names_partition, subquery_name) - .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE)) + pkgs = Packages::Package + pkgs = pkgs.with(project_ids_cte.to_arel) if use_project_ids_cte? + pkgs = pkgs.select(column_names.join(',')) + .from(package_names_partition, subquery_name) + .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE)) return pkgs if include_prerelease_versions? # we can't use pkgs.without_version_like since we have a custom from pkgs.where.not(arel_table[:version].matches(PRE_RELEASE_VERSION_MATCHING_TERM)) + # rubocop: enable CodeReuse/ActiveRecord end def package_names_partition + # rubocop: disable CodeReuse/ActiveRecord table_name = quote_table_name(Packages::Package.table_name) name_column = "#{table_name}.#{quote_column_name('name')}" created_at_column = "#{table_name}.#{quote_column_name('created_at')}" select_sql = "ROW_NUMBER() OVER (PARTITION BY #{name_column} ORDER BY #{created_at_column} DESC) AS row_number, #{table_name}.*" - @project.packages - .select(select_sql) - .nuget - .has_version - .without_nuget_temporary_name - .with_name(package_names) + nuget_packages.select(select_sql) + .with_name(paginated_matching_package_names) + .where(project_id: project_ids) + # rubocop: enable CodeReuse/ActiveRecord end - def package_names - strong_memoize(:package_names) do - pkgs = @project.packages - .nuget - .has_version - .without_nuget_temporary_name - .order_name - .select_distinct_name + def paginated_matching_package_names + pkgs = base_matching_package_names + pkgs.page(0) # we're using a padding + .per(per_page) + .padding(padding) + end + + def non_paginated_matching_package_names + # rubocop: disable CodeReuse/ActiveRecord + pkgs = base_matching_package_names + pkgs = pkgs.with(project_ids_cte.to_arel) if use_project_ids_cte? + pkgs + # rubocop: enable CodeReuse/ActiveRecord + end + + def base_matching_package_names + strong_memoize(:base_matching_package_names) do + # rubocop: disable CodeReuse/ActiveRecord + pkgs = nuget_packages.order_name + .select_distinct_name + .where(project_id: project_ids) pkgs = pkgs.without_version_like(PRE_RELEASE_VERSION_MATCHING_TERM) unless include_prerelease_versions? pkgs = pkgs.search_by_name(@search_term) if @search_term.present? - pkgs.page(0) # we're using a padding - .per(per_page) - .padding(padding) + pkgs + # rubocop: enable CodeReuse/ActiveRecord end end + def nuget_packages + Packages::Package.nuget + .has_version + .without_nuget_temporary_name + end + + def project_ids_cte + return unless use_project_ids_cte? + + strong_memoize(:project_ids_cte) do + query = projects_visible_to_user(@current_user, within_group: @project_or_group) + Gitlab::SQL::CTE.new(:project_ids, query.select(:id)) + end + end + + def project_ids + return @project_or_group.id if project? + + if use_project_ids_cte? + # rubocop: disable CodeReuse/ActiveRecord + Project.select(:id) + .from(project_ids_cte.table) + # rubocop: enable CodeReuse/ActiveRecord + end + end + + def use_project_ids_cte? + group? + end + + def project? + @project_or_group.is_a?(::Project) + end + + def group? + @project_or_group.is_a?(::Group) + end + def include_prerelease_versions? @options[:include_prerelease_versions] end @@ -96,6 +150,12 @@ module Packages def per_page [@options[:per_page], MAX_PER_PAGE].min end + + class Result + include ActiveModel::Model + + attr_accessor :results, :total_count + end end end end diff --git a/app/services/pages/migrate_legacy_storage_to_deployment_service.rb b/app/services/pages/migrate_legacy_storage_to_deployment_service.rb new file mode 100644 index 00000000000..dac994b2ccc --- /dev/null +++ b/app/services/pages/migrate_legacy_storage_to_deployment_service.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Pages + class MigrateLegacyStorageToDeploymentService + ExclusiveLeaseTakenError = Class.new(StandardError) + + include BaseServiceUtility + include ::Pages::LegacyStorageLease + + attr_reader :project + + def initialize(project) + @project = project + end + + def execute + result = try_obtain_lease do + execute_unsafe + end + + raise ExclusiveLeaseTakenError, "Can't migrate pages for project #{project.id}: exclusive lease taken" if result.nil? + + result + end + + private + + def execute_unsafe + zip_result = ::Pages::ZipDirectoryService.new(project.pages_path).execute + + if zip_result[:status] == :error + if !project.pages_metadatum&.reload&.pages_deployment && + Feature.enabled?(:pages_migration_mark_as_not_deployed, project) + project.mark_pages_as_not_deployed + end + + return error("Can't create zip archive: #{zip_result[:message]}") + end + + archive_path = zip_result[:archive_path] + + deployment = nil + File.open(archive_path) do |file| + deployment = project.pages_deployments.create!( + file: file, + file_count: zip_result[:entries_count], + file_sha256: Digest::SHA256.file(archive_path).hexdigest + ) + end + + project.set_first_pages_deployment!(deployment) + + success + ensure + FileUtils.rm_f(archive_path) if archive_path + end + end +end diff --git a/app/services/pages/zip_directory_service.rb b/app/services/pages/zip_directory_service.rb index a27ad5fda46..ba7a8571e88 100644 --- a/app/services/pages/zip_directory_service.rb +++ b/app/services/pages/zip_directory_service.rb @@ -2,37 +2,43 @@ module Pages class ZipDirectoryService - InvalidArchiveError = Class.new(RuntimeError) - InvalidEntryError = Class.new(RuntimeError) + include BaseServiceUtility + include Gitlab::Utils::StrongMemoize + + # used only to track exceptions in Sentry + InvalidEntryError = Class.new(StandardError) PUBLIC_DIR = 'public' def initialize(input_dir) - @input_dir = File.realpath(input_dir) - @output_file = File.join(@input_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects + @input_dir = input_dir end def execute - FileUtils.rm_f(@output_file) + return error("Can not find valid public dir in #{@input_dir}") unless valid_path?(public_dir) + + output_file = File.join(real_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects + + FileUtils.rm_f(output_file) - count = 0 - ::Zip::File.open(@output_file, ::Zip::File::CREATE) do |zipfile| + entries_count = 0 + ::Zip::File.open(output_file, ::Zip::File::CREATE) do |zipfile| write_entry(zipfile, PUBLIC_DIR) - count = zipfile.entries.count + entries_count = zipfile.entries.count end - [@output_file, count] + success(archive_path: output_file, entries_count: entries_count) + rescue => e + FileUtils.rm_f(output_file) if output_file + raise e end private def write_entry(zipfile, zipfile_path) - disk_file_path = File.join(@input_dir, zipfile_path) + disk_file_path = File.join(real_dir, zipfile_path) unless valid_path?(disk_file_path) - # archive without public directory is completelly unusable - raise InvalidArchiveError if zipfile_path == PUBLIC_DIR - # archive with invalid entry will just have this entry missing raise InvalidEntryError end @@ -71,13 +77,24 @@ module Pages def valid_path?(disk_file_path) realpath = File.realpath(disk_file_path) - realpath == File.join(@input_dir, PUBLIC_DIR) || - realpath.start_with?(File.join(@input_dir, PUBLIC_DIR + "/")) + realpath == public_dir || realpath.start_with?(public_dir + "/") # happens if target of symlink isn't there rescue => e - Gitlab::ErrorTracking.track_exception(e, input_dir: @input_dir, disk_file_path: disk_file_path) + Gitlab::ErrorTracking.track_exception(e, input_dir: real_dir, disk_file_path: disk_file_path) false end + + def real_dir + strong_memoize(:real_dir) do + File.realpath(@input_dir) rescue nil + end + end + + def public_dir + strong_memoize(:public_dir) do + File.join(real_dir, PUBLIC_DIR) rescue nil + end + end end end diff --git a/app/services/post_receive_service.rb b/app/services/post_receive_service.rb index bd9588844ad..84d9db5435b 100644 --- a/app/services/post_receive_service.rb +++ b/app/services/post_receive_service.rb @@ -94,7 +94,7 @@ class PostReceiveService end def record_onboarding_progress - NamespaceOnboardingAction.create_action(project.namespace, :git_write) + OnboardingProgressService.new(project.namespace).execute(action: :git_write) end end diff --git a/app/services/projects/after_import_service.rb b/app/services/projects/after_import_service.rb index b37ae56ba0f..bb0d084d191 100644 --- a/app/services/projects/after_import_service.rb +++ b/app/services/projects/after_import_service.rb @@ -9,7 +9,7 @@ module Projects end def execute - service = Projects::HousekeepingService.new(@project) + service = Repositories::HousekeepingService.new(@project) service.execute do import_failure_service.with_retry(action: 'delete_all_refs') do @@ -21,7 +21,7 @@ module Projects # import actually changed, so we increment the counter to avoid # causing GC to run every time. service.increment! - rescue Projects::HousekeepingService::LeaseTaken => e + rescue Repositories::HousekeepingService::LeaseTaken => e Gitlab::Import::Logger.info( message: 'Project housekeeping failed', project_full_path: @project.full_path, diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index b8047a1ad71..af0107436c8 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -8,12 +8,26 @@ module Projects return error('invalid regex') unless valid_regex? tags = container_repository.tags + original_size = tags.size + tags = without_latest(tags) tags = filter_by_name(tags) + + before_truncate_size = tags.size + tags = truncate(tags) + after_truncate_size = tags.size + tags = filter_keep_n(tags) tags = filter_by_older_than(tags) - delete_tags(container_repository, tags) + delete_tags(container_repository, tags).tap do |result| + result[:original_size] = original_size + result[:before_truncate_size] = before_truncate_size + result[:after_truncate_size] = after_truncate_size + result[:before_delete_size] = tags.size + + result[:status] = :error if before_truncate_size != after_truncate_size + end end private @@ -23,12 +37,14 @@ module Projects tag_names = tags.map(&:name) - Projects::ContainerRepository::DeleteTagsService - .new(container_repository.project, - current_user, - tags: tag_names, - container_expiration_policy: params['container_expiration_policy']) - .execute(container_repository) + service = Projects::ContainerRepository::DeleteTagsService.new( + container_repository.project, + current_user, + tags: tag_names, + container_expiration_policy: params['container_expiration_policy'] + ) + + service.execute(container_repository) end def without_latest(tags) @@ -54,7 +70,7 @@ module Projects return tags unless params['keep_n'] tags = order_by_date(tags) - tags.drop(params['keep_n'].to_i) + tags.drop(keep_n) end def filter_by_older_than(tags) @@ -83,6 +99,31 @@ module Projects ::Gitlab::ErrorTracking.log_exception(e, project_id: project.id) false end + + def truncate(tags) + return tags unless throttling_enabled? + return tags if max_list_size == 0 + + # truncate the list to make sure that after the #filter_keep_n + # execution, the resulting list will be max_list_size + truncated_size = max_list_size + keep_n + + return tags if tags.size <= truncated_size + + tags.sample(truncated_size) + end + + def throttling_enabled? + Feature.enabled?(:container_registry_expiration_policies_throttling) + end + + def max_list_size + ::Gitlab::CurrentSettings.current_application_settings.container_registry_cleanup_tags_service_max_list_size.to_i + end + + def keep_n + params['keep_n'].to_i + end end end end diff --git a/app/services/projects/container_repository/gitlab/delete_tags_service.rb b/app/services/projects/container_repository/gitlab/delete_tags_service.rb index e4e22dd9543..589aac5c3ac 100644 --- a/app/services/projects/container_repository/gitlab/delete_tags_service.rb +++ b/app/services/projects/container_repository/gitlab/delete_tags_service.rb @@ -24,9 +24,9 @@ module Projects return success(deleted: []) if @tag_names.empty? delete_tags - rescue TimeoutError => e + rescue TimeoutError, ::Faraday::Error => e ::Gitlab::ErrorTracking.track_exception(e, tags_count: @tag_names&.size, container_repository_id: @container_repository&.id) - error('timeout while deleting tags', nil, pass_back: { deleted: @deleted_tags }) + error('error while deleting tags', nil, pass_back: { deleted: @deleted_tags, exception_class_name: e.class.name }) end private diff --git a/app/services/projects/fork_service.rb b/app/services/projects/fork_service.rb index 0b4963e356a..050bfdd862d 100644 --- a/app/services/projects/fork_service.rb +++ b/app/services/projects/fork_service.rb @@ -34,11 +34,6 @@ module Projects new_project = CreateService.new(current_user, new_fork_params).execute return new_project unless new_project.persisted? - # Set the forked_from_project relation after saving to avoid having to - # reload the project to reset the association information and cause an - # extra query. - new_project.forked_from_project = @project - builds_access_level = @project.project_feature.builds_access_level new_project.project_feature.update(builds_access_level: builds_access_level) @@ -47,6 +42,7 @@ module Projects def new_fork_params new_params = { + forked_from_project: @project, visibility_level: allowed_visibility_level, description: @project.description, name: target_name, diff --git a/app/services/projects/housekeeping_service.rb b/app/services/projects/housekeeping_service.rb index 9428575591e..b5589d556aa 100644 --- a/app/services/projects/housekeeping_service.rb +++ b/app/services/projects/housekeeping_service.rb @@ -1,107 +1,16 @@ # frozen_string_literal: true -# Projects::HousekeepingService class +# This is a compatibility class to avoid calling a non-existent +# class from sidekiq during deployment. # -# Used for git housekeeping +# We're deploying the rename of this class in 13.9. Nevertheless, +# we cannot remove this class entirely because there can be jobs +# referencing it. # -# Ex. -# Projects::HousekeepingService.new(project).execute +# We can get rid of this class in 13.10 +# https://gitlab.com/gitlab-org/gitlab/-/issues/297580 # module Projects - class HousekeepingService < BaseService - # Timeout set to 24h - LEASE_TIMEOUT = 86400 - PACK_REFS_PERIOD = 6 - - class LeaseTaken < StandardError - def to_s - "Somebody already triggered housekeeping for this project in the past #{LEASE_TIMEOUT / 60} minutes" - end - end - - def initialize(project, task = nil) - @project = project - @task = task - end - - def execute - lease_uuid = try_obtain_lease - raise LeaseTaken unless lease_uuid.present? - - yield if block_given? - - execute_gitlab_shell_gc(lease_uuid) - end - - def needed? - pushes_since_gc > 0 && period_match? && housekeeping_enabled? - end - - def increment! - Gitlab::Metrics.measure(:increment_pushes_since_gc) do - @project.increment_pushes_since_gc - end - end - - private - - def execute_gitlab_shell_gc(lease_uuid) - GitGarbageCollectWorker.perform_async(@project.id, task, lease_key, lease_uuid) - ensure - if pushes_since_gc >= gc_period - Gitlab::Metrics.measure(:reset_pushes_since_gc) do - @project.reset_pushes_since_gc - end - end - end - - def try_obtain_lease - Gitlab::Metrics.measure(:obtain_housekeeping_lease) do - lease = ::Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT) - lease.try_obtain - end - end - - def lease_key - "project_housekeeping:#{@project.id}" - end - - def pushes_since_gc - @project.pushes_since_gc - end - - def task - return @task if @task - - if pushes_since_gc % gc_period == 0 - :gc - elsif pushes_since_gc % full_repack_period == 0 - :full_repack - elsif pushes_since_gc % repack_period == 0 - :incremental_repack - else - :pack_refs - end - end - - def period_match? - [gc_period, full_repack_period, repack_period, PACK_REFS_PERIOD].any? { |period| pushes_since_gc % period == 0 } - end - - def housekeeping_enabled? - Gitlab::CurrentSettings.housekeeping_enabled - end - - def gc_period - Gitlab::CurrentSettings.housekeeping_gc_period - end - - def full_repack_period - Gitlab::CurrentSettings.housekeeping_full_repack_period - end - - def repack_period - Gitlab::CurrentSettings.housekeeping_incremental_repack_period - end + class HousekeepingService < ::Repositories::HousekeepingService end end diff --git a/app/services/projects/schedule_bulk_repository_shard_moves_service.rb b/app/services/projects/schedule_bulk_repository_shard_moves_service.rb index dd49910207f..53de9abdb59 100644 --- a/app/services/projects/schedule_bulk_repository_shard_moves_service.rb +++ b/app/services/projects/schedule_bulk_repository_shard_moves_service.rb @@ -3,33 +3,29 @@ module Projects # Tries to schedule a move for every project with repositories on the source shard class ScheduleBulkRepositoryShardMovesService - include BaseServiceUtility + include ScheduleBulkRepositoryShardMovesMethods + extend ::Gitlab::Utils::Override - def execute(source_storage_name, destination_storage_name = nil) - shard = Shard.find_by_name!(source_storage_name) + private - ProjectRepository.for_shard(shard).each_batch(column: :project_id) do |relation| - Project.id_in(relation.select(:project_id)).each do |project| - project.with_lock do - next if project.repository_storage != source_storage_name - - storage_move = project.repository_storage_moves.build( - source_storage_name: source_storage_name, - destination_storage_name: destination_storage_name - ) + override :repository_klass + def repository_klass + ProjectRepository + end - unless storage_move.schedule - log_info("Project #{project.full_path} (#{project.id}) was skipped: #{storage_move.errors.full_messages.to_sentence}") - end - end - end - end + override :container_klass + def container_klass + Project + end - success + override :container_column + def container_column + :project_id end - def self.enqueue(source_storage_name, destination_storage_name = nil) - ::ProjectScheduleBulkRepositoryShardMovesWorker.perform_async(source_storage_name, destination_storage_name) + override :schedule_bulk_worker_klass + def self.schedule_bulk_worker_klass + ::ProjectScheduleBulkRepositoryShardMovesWorker end end end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 1574c90d2ac..8a5e0706126 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -37,7 +37,7 @@ module Projects private - attr_reader :old_path, :new_path, :new_namespace + attr_reader :old_path, :new_path, :new_namespace, :old_namespace # rubocop: disable CodeReuse/ActiveRecord def transfer(project) @@ -96,7 +96,7 @@ module Projects execute_system_hooks end - move_pages(project) + post_update_hooks(project) rescue Exception # rubocop:disable Lint/RescueException rollback_side_effects raise @@ -104,6 +104,11 @@ module Projects refresh_permissions end + # Overridden in EE + def post_update_hooks(project) + move_pages(project) + end + def transfer_missing_group_resources(group) Labels::TransferService.new(current_user, group, project).execute diff --git a/app/services/projects/unlink_fork_service.rb b/app/services/projects/unlink_fork_service.rb index 52aea8c51a5..6ba3356d612 100644 --- a/app/services/projects/unlink_fork_service.rb +++ b/app/services/projects/unlink_fork_service.rb @@ -9,6 +9,8 @@ module Projects return unless fork_network + log_info(message: "UnlinkForkService: Unlinking fork network", fork_network_id: fork_network.id) + merge_requests = fork_network .merge_requests .opened @@ -16,6 +18,7 @@ module Projects merge_requests.find_each do |mr| ::MergeRequests::CloseService.new(@project, @current_user).execute(mr) + log_info(message: "UnlinkForkService: Closed merge request", merge_request_id: mr.id) end Project.transaction do @@ -31,6 +34,16 @@ module Projects end end + # rubocop: disable Cop/InBatches + Project.uncached do + @project.forked_to_members.in_batches do |fork_relation| + fork_relation.pluck(:id).each do |fork_id| # rubocop: disable CodeReuse/ActiveRecord + log_info(message: "UnlinkForkService: Unlinked fork of root_project", project_id: @project.id, forked_project_id: fork_id) + end + end + end + # rubocop: enable Cop/InBatches + # When the project getting out of the network is a node with parent # and children, both the parent and the node needs a cache refresh. [forked_from, @project].compact.each do |project| diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index 53872c67f49..25d46ada885 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -37,16 +37,11 @@ module Projects raise InvalidStateError, 'missing pages artifacts' unless build.artifacts? raise InvalidStateError, 'build SHA is outdated for this ref' unless latest? - # Create temporary directory in which we will extract the artifacts - make_secure_tmp_dir(tmp_path) do |archive_path| - extract_archive!(archive_path) + build.artifacts_file.use_file do |artifacts_path| + deploy_to_legacy_storage(artifacts_path) - # Check if we did extract public directory - archive_public_path = File.join(archive_path, PUBLIC_DIR) - raise InvalidStateError, 'pages miss the public folder' unless Dir.exist?(archive_public_path) - raise InvalidStateError, 'build SHA is outdated for this ref' unless latest? + create_pages_deployment(artifacts_path, build) - deploy_page!(archive_public_path) success end rescue InvalidStateError => e @@ -84,15 +79,29 @@ module Projects ) end - def extract_archive!(temp_path) + def deploy_to_legacy_storage(artifacts_path) + # Create temporary directory in which we will extract the artifacts + make_secure_tmp_dir(tmp_path) do |tmp_path| + extract_archive!(artifacts_path, tmp_path) + + # Check if we did extract public directory + archive_public_path = File.join(tmp_path, PUBLIC_DIR) + raise InvalidStateError, 'pages miss the public folder' unless Dir.exist?(archive_public_path) + raise InvalidStateError, 'build SHA is outdated for this ref' unless latest? + + deploy_page!(archive_public_path) + end + end + + def extract_archive!(artifacts_path, temp_path) if artifacts.ends_with?('.zip') - extract_zip_archive!(temp_path) + extract_zip_archive!(artifacts_path, temp_path) else raise InvalidStateError, 'unsupported artifacts format' end end - def extract_zip_archive!(temp_path) + def extract_zip_archive!(artifacts_path, temp_path) raise InvalidStateError, 'missing artifacts metadata' unless build.artifacts_metadata? # Calculate page size after extract @@ -102,11 +111,8 @@ module Projects raise InvalidStateError, "artifacts for pages are too large: #{public_entry.total_size}" end - build.artifacts_file.use_file do |artifacts_path| - SafeZip::Extract.new(artifacts_path) - .extract(directories: [PUBLIC_DIR], to: temp_path) - create_pages_deployment(artifacts_path, build) - end + SafeZip::Extract.new(artifacts_path) + .extract(directories: [PUBLIC_DIR], to: temp_path) rescue SafeZip::Extract::Error => e raise FailedToExtractError, e.message end @@ -150,6 +156,9 @@ module Projects deployment = project.pages_deployments.create!(file: file, file_count: entries_count, file_sha256: sha256) + + raise InvalidStateError, 'build SHA is outdated for this ref' unless latest? + project.update_pages_deployment!(deployment) end diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb index e0d2398bc66..7c63216af5e 100644 --- a/app/services/projects/update_repository_storage_service.rb +++ b/app/services/projects/update_repository_storage_service.rb @@ -2,59 +2,19 @@ module Projects class UpdateRepositoryStorageService - Error = Class.new(StandardError) - SameFilesystemError = Class.new(Error) + include UpdateRepositoryStorageMethods - attr_reader :repository_storage_move - delegate :project, :source_storage_name, :destination_storage_name, to: :repository_storage_move - - def initialize(repository_storage_move) - @repository_storage_move = repository_storage_move - end - - def execute - repository_storage_move.with_lock do - return ServiceResponse.success unless repository_storage_move.scheduled? # rubocop:disable Cop/AvoidReturnFromBlocks - - repository_storage_move.start! - end - - raise SameFilesystemError if same_filesystem?(source_storage_name, destination_storage_name) - - mirror_repositories - - repository_storage_move.transaction do - repository_storage_move.finish_replication! - - project.leave_pool_repository - project.track_project_repository - end - - remove_old_paths - enqueue_housekeeping - - repository_storage_move.finish_cleanup! - - ServiceResponse.success - - rescue StandardError => e - repository_storage_move.do_fail! - - Gitlab::ErrorTracking.track_exception(e, project_path: project.full_path) - - ServiceResponse.error( - message: s_("UpdateRepositoryStorage|Error moving repository storage for %{project_full_path} - %{message}") % { project_full_path: project.full_path, message: e.message } - ) - end + delegate :project, to: :repository_storage_move private - def same_filesystem?(old_storage, new_storage) - Gitlab::GitalyClient.filesystem_id(old_storage) == Gitlab::GitalyClient.filesystem_id(new_storage) + def track_repository(_destination_storage_name) + project.leave_pool_repository + project.track_project_repository end def mirror_repositories - mirror_repository if project.repository_exists? + mirror_repository(type: Gitlab::GlRepository::PROJECT) if project.repository_exists? if project.wiki.repository_exists? mirror_repository(type: Gitlab::GlRepository::WIKI) @@ -65,41 +25,21 @@ module Projects end end - def mirror_repository(type: Gitlab::GlRepository::PROJECT) - unless wait_for_pushes(type) - raise Error, s_('UpdateRepositoryStorage|Timeout waiting for %{type} repository pushes') % { type: type.name } - end - - repository = type.repository_for(project) - full_path = repository.full_path - raw_repository = repository.raw - checksum = repository.checksum - - # Initialize a git repository on the target path - new_repository = Gitlab::Git::Repository.new( - destination_storage_name, - raw_repository.relative_path, - raw_repository.gl_repository, - full_path - ) - - new_repository.replicate(raw_repository) - new_checksum = new_repository.checksum + # The underlying FetchInternalRemote call uses a `git fetch` to move data + # to the new repository, which leaves it in a less-well-packed state, + # lacking bitmaps and commit graphs. Housekeeping will boost performance + # significantly. + def enqueue_housekeeping + return unless Gitlab::CurrentSettings.housekeeping_enabled? + return unless Feature.enabled?(:repack_after_shard_migration, project) - if checksum != new_checksum - raise Error, s_('UpdateRepositoryStorage|Failed to verify %{type} repository checksum from %{old} to %{new}') % { type: type.name, old: checksum, new: new_checksum } - end + Repositories::HousekeepingService.new(project, :gc).execute + rescue Repositories::HousekeepingService::LeaseTaken + # No action required end def remove_old_paths - if project.repository_exists? - Gitlab::Git::Repository.new( - source_storage_name, - "#{project.disk_path}.git", - nil, - nil - ).remove - end + super if project.wiki.repository_exists? Gitlab::Git::Repository.new( @@ -119,31 +59,5 @@ module Projects ).remove end end - - # The underlying FetchInternalRemote call uses a `git fetch` to move data - # to the new repository, which leaves it in a less-well-packed state, - # lacking bitmaps and commit graphs. Housekeeping will boost performance - # significantly. - def enqueue_housekeeping - return unless Gitlab::CurrentSettings.housekeeping_enabled? - return unless Feature.enabled?(:repack_after_shard_migration, project) - - Projects::HousekeepingService.new(project, :gc).execute - rescue Projects::HousekeepingService::LeaseTaken - # No action required - end - - def wait_for_pushes(type) - reference_counter = project.reference_counter(type: type) - - # Try for 30 seconds, polling every 10 - 3.times do - return true if reference_counter.value == 0 - - sleep 10 - end - - false - end end end diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index d44f5e637f1..50a544ed1a5 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -87,11 +87,6 @@ module Projects system_hook_service.execute_hooks_for(project, :update) end - if project.visibility_level_decreased? && project.unlink_forks_upon_visibility_decrease_enabled? - # It's a system-bounded operation, so no extra authorization check is required. - Projects::UnlinkForkService.new(project, current_user).execute - end - update_pages_config if changing_pages_related_config? end diff --git a/app/services/repositories/housekeeping_service.rb b/app/services/repositories/housekeeping_service.rb new file mode 100644 index 00000000000..6a2fa95d25f --- /dev/null +++ b/app/services/repositories/housekeeping_service.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +# Used for git housekeeping +# +# Ex. +# Repositories::HousekeepingService.new(project).execute +# Repositories::HousekeepingService.new(project.wiki).execute +# +module Repositories + class HousekeepingService < BaseService + # Timeout set to 24h + LEASE_TIMEOUT = 86400 + PACK_REFS_PERIOD = 6 + + class LeaseTaken < StandardError + def to_s + "Somebody already triggered housekeeping for this resource in the past #{LEASE_TIMEOUT / 60} minutes" + end + end + + def initialize(resource, task = nil) + @resource = resource + @task = task + end + + def execute + lease_uuid = try_obtain_lease + raise LeaseTaken unless lease_uuid.present? + + yield if block_given? + + execute_gitlab_shell_gc(lease_uuid) + end + + def needed? + pushes_since_gc > 0 && period_match? && housekeeping_enabled? + end + + def increment! + Gitlab::Metrics.measure(:increment_pushes_since_gc) do + @resource.increment_pushes_since_gc + end + end + + private + + def execute_gitlab_shell_gc(lease_uuid) + GitGarbageCollectWorker.perform_async(@resource.id, task, lease_key, lease_uuid) + ensure + if pushes_since_gc >= gc_period + Gitlab::Metrics.measure(:reset_pushes_since_gc) do + @resource.reset_pushes_since_gc + end + end + end + + def try_obtain_lease + Gitlab::Metrics.measure(:obtain_housekeeping_lease) do + lease = ::Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT) + lease.try_obtain + end + end + + def lease_key + "#{@resource.class.name.underscore.pluralize}_housekeeping:#{@resource.id}" + end + + def pushes_since_gc + @resource.pushes_since_gc + end + + def task + return @task if @task + + if pushes_since_gc % gc_period == 0 + :gc + elsif pushes_since_gc % full_repack_period == 0 + :full_repack + elsif pushes_since_gc % repack_period == 0 + :incremental_repack + else + :pack_refs + end + end + + def period_match? + [gc_period, full_repack_period, repack_period, PACK_REFS_PERIOD].any? { |period| pushes_since_gc % period == 0 } + end + + def housekeeping_enabled? + Gitlab::CurrentSettings.housekeeping_enabled + end + + def gc_period + Gitlab::CurrentSettings.housekeeping_gc_period + end + + def full_repack_period + Gitlab::CurrentSettings.housekeeping_full_repack_period + end + + def repack_period + Gitlab::CurrentSettings.housekeeping_incremental_repack_period + end + end +end diff --git a/app/services/resource_events/change_state_service.rb b/app/services/resource_events/change_state_service.rb index cd6d82df46f..c5120ba82e1 100644 --- a/app/services/resource_events/change_state_service.rb +++ b/app/services/resource_events/change_state_service.rb @@ -19,7 +19,7 @@ module ResourceEvents state: ResourceStateEvent.states[state], close_after_error_tracking_resolve: close_after_error_tracking_resolve, close_auto_resolve_prometheus_alert: close_auto_resolve_prometheus_alert, - created_at: Time.zone.now + created_at: resource.system_note_timestamp ) resource.expire_note_etag_cache diff --git a/app/services/serverless/associate_domain_service.rb b/app/services/serverless/associate_domain_service.rb index 673f1f83260..0c6ee58924c 100644 --- a/app/services/serverless/associate_domain_service.rb +++ b/app/services/serverless/associate_domain_service.rb @@ -2,7 +2,7 @@ module Serverless class AssociateDomainService - PLACEHOLDER_HOSTNAME = 'example.com'.freeze + PLACEHOLDER_HOSTNAME = 'example.com' def initialize(knative, pages_domain_id:, creator:) @knative = knative diff --git a/app/services/service_desk_settings/update_service.rb b/app/services/service_desk_settings/update_service.rb index 32d1c5c1c87..5fe74f1f2ff 100644 --- a/app/services/service_desk_settings/update_service.rb +++ b/app/services/service_desk_settings/update_service.rb @@ -5,10 +5,6 @@ module ServiceDeskSettings def execute settings = ServiceDeskSetting.safe_find_or_create_by!(project_id: project.id) - unless ::Feature.enabled?(:service_desk_custom_address, project, default_enabled: true) - params.delete(:project_key) - end - params[:project_key] = nil if params[:project_key].blank? if settings.update(params) diff --git a/app/services/snippets/schedule_bulk_repository_shard_moves_service.rb b/app/services/snippets/schedule_bulk_repository_shard_moves_service.rb new file mode 100644 index 00000000000..f7bdd0a99a5 --- /dev/null +++ b/app/services/snippets/schedule_bulk_repository_shard_moves_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Snippets + # Tries to schedule a move for every snippet with repositories on the source shard + class ScheduleBulkRepositoryShardMovesService + include ScheduleBulkRepositoryShardMovesMethods + extend ::Gitlab::Utils::Override + + private + + override :repository_klass + def repository_klass + SnippetRepository + end + + override :container_klass + def container_klass + Snippet + end + + override :container_column + def container_column + :snippet_id + end + + override :schedule_bulk_worker_klass + def self.schedule_bulk_worker_klass + ::SnippetScheduleBulkRepositoryShardMovesWorker + end + end +end diff --git a/app/services/snippets/update_repository_storage_service.rb b/app/services/snippets/update_repository_storage_service.rb new file mode 100644 index 00000000000..3addae3b3be --- /dev/null +++ b/app/services/snippets/update_repository_storage_service.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Snippets + class UpdateRepositoryStorageService + include UpdateRepositoryStorageMethods + + delegate :snippet, to: :repository_storage_move + + private + + def track_repository(destination_storage_name) + snippet.track_snippet_repository(destination_storage_name) + end + + def mirror_repositories + return unless snippet.repository_exists? + + mirror_repository(type: Gitlab::GlRepository::SNIPPET) + end + end +end diff --git a/app/services/todo_service.rb b/app/services/todo_service.rb index 0eb099753cb..12d26fe890b 100644 --- a/app/services/todo_service.rb +++ b/app/services/todo_service.rb @@ -216,7 +216,7 @@ class TodoService def create_todos(users, attributes) Array(users).map do |user| - next if pending_todos(user, attributes).exists? + next if pending_todos(user, attributes).exists? && Feature.disabled?(:multiple_todos, user) issue_type = attributes.delete(:issue_type) track_todo_creation(user, issue_type) @@ -278,7 +278,7 @@ class TodoService create_todos(directly_addressed_users, attributes) # Create Todos for mentioned users - mentioned_users = filter_mentioned_users(parent, note || target, author, skip_users) + mentioned_users = filter_mentioned_users(parent, note || target, author, skip_users + directly_addressed_users) attributes = attributes_for_todo(parent, target, author, Todo::MENTIONED, note) create_todos(mentioned_users, attributes) end diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index aef07b13cae..5a51b42f9f9 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -54,7 +54,9 @@ class WebHookService http_status: response.code, message: response.to_s } - rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep, Gitlab::Json::LimitedEncoder::LimitExceeded => e + rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, + Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep, + Gitlab::Json::LimitedEncoder::LimitExceeded, URI::InvalidURIError => e execution_duration = Gitlab::Metrics::System.monotonic_time - start_time log_execution( trigger: hook_name, |