diff options
Diffstat (limited to 'app/services/projects')
20 files changed, 141 insertions, 197 deletions
diff --git a/app/services/projects/batch_count_service.rb b/app/services/projects/batch_count_service.rb index aec3b32da89..455c7211ab2 100644 --- a/app/services/projects/batch_count_service.rb +++ b/app/services/projects/batch_count_service.rb @@ -9,13 +9,19 @@ module Projects @projects = projects end - def refresh_cache - @projects.each do |project| - service = count_service.new(project) - unless service.count_stored? - service.refresh_cache { global_count[project.id].to_i } + def refresh_cache_and_retrieve_data + count_services = @projects.map { |project| count_service.new(project) } + services_by_cache_key = count_services.index_by(&:cache_key) + + results = Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Rails.cache.fetch_multi(*services_by_cache_key.keys) do |key| + service = services_by_cache_key[key] + + global_count[service.project.id].to_i end end + + results.transform_keys! { |cache_key| services_by_cache_key[cache_key].project } end def project_ids diff --git a/app/services/projects/batch_forks_count_service.rb b/app/services/projects/batch_forks_count_service.rb index d12772b40ff..6467744a435 100644 --- a/app/services/projects/batch_forks_count_service.rb +++ b/app/services/projects/batch_forks_count_service.rb @@ -5,21 +5,6 @@ # because the service use maps to retrieve the project ids module Projects class BatchForksCountService < Projects::BatchCountService - def refresh_cache_and_retrieve_data - count_services = @projects.map { |project| count_service.new(project) } - - values = Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - Rails.cache.fetch_multi(*(count_services.map { |ser| ser.cache_key } )) { |key| nil } - end - - results_per_service = Hash[count_services.zip(values.values)] - projects_to_refresh = results_per_service.select { |_k, value| value.nil? } - projects_to_refresh = recreate_cache(projects_to_refresh) - - results_per_service.update(projects_to_refresh) - results_per_service.transform_keys { |k| k.project } - end - # rubocop: disable CodeReuse/ActiveRecord def global_count @global_count ||= begin @@ -33,13 +18,5 @@ module Projects def count_service ::Projects::ForksCountService end - - def recreate_cache(projects_to_refresh) - projects_to_refresh.each_with_object({}) do |(service, _v), hash| - count = global_count[service.project.id].to_i - service.refresh_cache { count } - hash[service] = count - end - end end end diff --git a/app/services/projects/count_service.rb b/app/services/projects/count_service.rb index 3cee80c7bbc..5daea3a2600 100644 --- a/app/services/projects/count_service.rb +++ b/app/services/projects/count_service.rb @@ -9,6 +9,8 @@ module Projects # all caches. VERSION = 1 + attr_reader :project + def initialize(project) @project = project end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 302c047a65f..e717491b19d 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -12,6 +12,7 @@ module Projects @import_data = @params.delete(:import_data) @relations_block = @params.delete(:relations_block) @default_branch = @params.delete(:default_branch) + @readme_template = @params.delete(:readme_template) build_topics end @@ -89,10 +90,14 @@ module Projects def after_create_actions log_info("#{@project.owner.name} created a new project \"#{@project.full_name}\"") - # Skip writing the config for project imports/forks because it - # will always fail since the Git directory doesn't exist until - # a background job creates it (see Project#add_import_job). - @project.set_full_path unless @project.import? + if @project.import? + experiment(:combined_registration, user: current_user).track(:import_project) + else + # Skip writing the config for project imports/forks because it + # will always fail since the Git directory doesn't exist until + # a background job creates it (see Project#add_import_job). + @project.set_full_path + end unless @project.gitlab_project_import? @project.create_wiki unless skip_wiki? @@ -149,12 +154,16 @@ module Projects branch_name: @default_branch.presence || @project.default_branch_or_main, commit_message: 'Initial commit', file_path: 'README.md', - file_content: experiment(:new_project_readme_content, namespace: @project.namespace).run_with(@project) + file_content: readme_content } Files::CreateService.new(@project, current_user, commit_attrs).execute end + def readme_content + @readme_template.presence || experiment(:new_project_readme_content, namespace: @project.namespace).run_with(@project) + end + def skip_wiki? !@project.feature_available?(:wiki, current_user) || @skip_wiki end diff --git a/app/services/projects/fork_service.rb b/app/services/projects/fork_service.rb index 3cee1b5975a..74b7d18f401 100644 --- a/app/services/projects/fork_service.rb +++ b/app/services/projects/fork_service.rb @@ -61,7 +61,8 @@ module Projects # initializing the project, as that would cause a foreign key constraint # exception. relations_block: -> (project) { build_fork_network_member(project) }, - skip_disk_validation: skip_disk_validation + skip_disk_validation: skip_disk_validation, + external_authorization_classification_label: @project.external_authorization_classification_label } if @project.avatar.present? && @project.avatar.image? diff --git a/app/services/projects/forks_count_service.rb b/app/services/projects/forks_count_service.rb index 848d8d54104..ca85e2dc281 100644 --- a/app/services/projects/forks_count_service.rb +++ b/app/services/projects/forks_count_service.rb @@ -3,8 +3,6 @@ module Projects # Service class for getting and caching the number of forks of a project. class ForksCountService < Projects::CountService - attr_reader :project - def cache_key_name 'forks_count' end diff --git a/app/services/projects/group_links/destroy_service.rb b/app/services/projects/group_links/destroy_service.rb index 01a5b617b46..19df0dc2c73 100644 --- a/app/services/projects/group_links/destroy_service.rb +++ b/app/services/projects/group_links/destroy_service.rb @@ -13,19 +13,15 @@ module Projects end group_link.destroy.tap do |link| - if Feature.enabled?(:use_specialized_worker_for_project_auth_recalculation) - refresh_project_authorizations_asynchronously(link.project) + refresh_project_authorizations_asynchronously(link.project) - # Until we compare the inconsistency rates of the new specialized worker and - # the old approach, we still run AuthorizedProjectsWorker - # but with some delay and lower urgency as a safety net. - link.group.refresh_members_authorized_projects( - blocking: false, - priority: UserProjectAccessChangedService::LOW_PRIORITY - ) - else - link.group.refresh_members_authorized_projects - end + # Until we compare the inconsistency rates of the new specialized worker and + # the old approach, we still run AuthorizedProjectsWorker + # but with some delay and lower urgency as a safety net. + link.group.refresh_members_authorized_projects( + blocking: false, + priority: UserProjectAccessChangedService::LOW_PRIORITY + ) end end diff --git a/app/services/projects/move_deploy_keys_projects_service.rb b/app/services/projects/move_deploy_keys_projects_service.rb index 51d84af249e..98ba5eb3f13 100644 --- a/app/services/projects/move_deploy_keys_projects_service.rb +++ b/app/services/projects/move_deploy_keys_projects_service.rb @@ -5,7 +5,7 @@ module Projects def execute(source_project, remove_remaining_elements: true) return unless super - Project.transaction(requires_new: true) do + Project.transaction do move_deploy_keys_projects remove_remaining_deploy_keys_projects if remove_remaining_elements diff --git a/app/services/projects/move_forks_service.rb b/app/services/projects/move_forks_service.rb index 33f0bab12c9..a96cf4dd3ea 100644 --- a/app/services/projects/move_forks_service.rb +++ b/app/services/projects/move_forks_service.rb @@ -5,7 +5,7 @@ module Projects def execute(source_project, remove_remaining_elements: true) return unless super && source_project.fork_network - Project.transaction(requires_new: true) do + Project.transaction do move_fork_network_members update_root_project refresh_forks_count diff --git a/app/services/projects/move_lfs_objects_projects_service.rb b/app/services/projects/move_lfs_objects_projects_service.rb index 57a8d3d69c6..7107ecc6c95 100644 --- a/app/services/projects/move_lfs_objects_projects_service.rb +++ b/app/services/projects/move_lfs_objects_projects_service.rb @@ -5,7 +5,7 @@ module Projects def execute(source_project, remove_remaining_elements: true) return unless super - Project.transaction(requires_new: true) do + Project.transaction do move_lfs_objects_projects remove_remaining_lfs_objects_project if remove_remaining_elements diff --git a/app/services/projects/move_notification_settings_service.rb b/app/services/projects/move_notification_settings_service.rb index efe06f158cc..fb84f10207d 100644 --- a/app/services/projects/move_notification_settings_service.rb +++ b/app/services/projects/move_notification_settings_service.rb @@ -5,7 +5,7 @@ module Projects def execute(source_project, remove_remaining_elements: true) return unless super - Project.transaction(requires_new: true) do + Project.transaction do move_notification_settings remove_remaining_notification_settings if remove_remaining_elements diff --git a/app/services/projects/move_project_authorizations_service.rb b/app/services/projects/move_project_authorizations_service.rb index c95ad60ab5e..6ac173a20fc 100644 --- a/app/services/projects/move_project_authorizations_service.rb +++ b/app/services/projects/move_project_authorizations_service.rb @@ -9,7 +9,7 @@ module Projects def execute(source_project, remove_remaining_elements: true) return unless super - Project.transaction(requires_new: true) do + Project.transaction do move_project_authorizations remove_remaining_authorizations if remove_remaining_elements diff --git a/app/services/projects/move_project_group_links_service.rb b/app/services/projects/move_project_group_links_service.rb index 349953ff973..5f6a7dd09e1 100644 --- a/app/services/projects/move_project_group_links_service.rb +++ b/app/services/projects/move_project_group_links_service.rb @@ -9,7 +9,7 @@ module Projects def execute(source_project, remove_remaining_elements: true) return unless super - Project.transaction(requires_new: true) do + Project.transaction do move_group_links remove_remaining_project_group_links if remove_remaining_elements diff --git a/app/services/projects/move_project_members_service.rb b/app/services/projects/move_project_members_service.rb index 9a1b7c6d1b6..011bd17c8cc 100644 --- a/app/services/projects/move_project_members_service.rb +++ b/app/services/projects/move_project_members_service.rb @@ -9,7 +9,7 @@ module Projects def execute(source_project, remove_remaining_elements: true) return unless super - Project.transaction(requires_new: true) do + Project.transaction do move_project_members remove_remaining_members if remove_remaining_elements diff --git a/app/services/projects/move_users_star_projects_service.rb b/app/services/projects/move_users_star_projects_service.rb index 20121d429e2..5490448553f 100644 --- a/app/services/projects/move_users_star_projects_service.rb +++ b/app/services/projects/move_users_star_projects_service.rb @@ -9,7 +9,7 @@ module Projects return unless user_stars.any? - Project.transaction(requires_new: true) do + Project.transaction do user_stars.update_all(project_id: @project.id) Project.reset_counters @project.id, :users_star_projects diff --git a/app/services/projects/open_issues_count_service.rb b/app/services/projects/open_issues_count_service.rb index dc450311db2..8b7a418edf5 100644 --- a/app/services/projects/open_issues_count_service.rb +++ b/app/services/projects/open_issues_count_service.rb @@ -7,8 +7,12 @@ module Projects include Gitlab::Utils::StrongMemoize # Cache keys used to store issues count - PUBLIC_COUNT_KEY = 'public_open_issues_count' - TOTAL_COUNT_KEY = 'total_open_issues_count' + # TOTAL_COUNT_KEY includes confidential and hidden issues (admin) + # TOTAL_COUNT_WITHOUT_HIDDEN_KEY includes confidential issues but not hidden issues (reporter and above) + # PUBLIC_COUNT_WITHOUT_HIDDEN_KEY does not include confidential or hidden issues (guest) + TOTAL_COUNT_KEY = 'project_open_issues_including_hidden_count' + TOTAL_COUNT_WITHOUT_HIDDEN_KEY = 'project_open_issues_without_hidden_count' + PUBLIC_COUNT_WITHOUT_HIDDEN_KEY = 'project_open_public_issues_without_hidden_count' def initialize(project, user = nil) @user = user @@ -16,16 +20,53 @@ module Projects super(project) end + # rubocop: disable CodeReuse/ActiveRecord + def refresh_cache(&block) + if block_given? + super(&block) + else + update_cache_for_key(total_count_cache_key) do + issues_with_hidden + end + + update_cache_for_key(public_count_without_hidden_cache_key) do + issues_without_hidden_without_confidential + end + + update_cache_for_key(total_count_without_hidden_cache_key) do + issues_without_hidden_with_confidential + end + end + end + + private + + def relation_for_count + self.class.query(@project, public_only: public_only?, include_hidden: include_hidden?) + end + def cache_key_name - public_only? ? PUBLIC_COUNT_KEY : TOTAL_COUNT_KEY + if include_hidden? + TOTAL_COUNT_KEY + elsif public_only? + PUBLIC_COUNT_WITHOUT_HIDDEN_KEY + else + TOTAL_COUNT_WITHOUT_HIDDEN_KEY + end + end + + def include_hidden? + user_is_admin? end def public_only? !user_is_at_least_reporter? end - def relation_for_count - self.class.query(@project, public_only: public_only?) + def user_is_admin? + strong_memoize(:user_is_admin) do + @user&.can_admin_all_resources? + end end def user_is_at_least_reporter? @@ -34,46 +75,43 @@ module Projects end end - def public_count_cache_key - cache_key(PUBLIC_COUNT_KEY) + def total_count_without_hidden_cache_key + cache_key(TOTAL_COUNT_WITHOUT_HIDDEN_KEY) + end + + def public_count_without_hidden_cache_key + cache_key(PUBLIC_COUNT_WITHOUT_HIDDEN_KEY) end def total_count_cache_key cache_key(TOTAL_COUNT_KEY) end - # rubocop: disable CodeReuse/ActiveRecord - def refresh_cache(&block) - if block_given? - super(&block) - else - count_grouped_by_confidential = self.class.query(@project, public_only: false).group(:confidential).count - public_count = count_grouped_by_confidential[false] || 0 - total_count = public_count + (count_grouped_by_confidential[true] || 0) + def issues_with_hidden + self.class.query(@project, public_only: false, include_hidden: true).count + end - update_cache_for_key(public_count_cache_key) do - public_count - end + def issues_without_hidden_without_confidential + self.class.query(@project, public_only: true, include_hidden: false).count + end - update_cache_for_key(total_count_cache_key) do - total_count - end - end + def issues_without_hidden_with_confidential + self.class.query(@project, public_only: false, include_hidden: false).count end - # rubocop: enable CodeReuse/ActiveRecord - # We only show total issues count for reporters - # which are allowed to view confidential issues + # We only show total issues count for admins, who are allowed to view hidden issues. + # We also only show issues count including confidential for reporters, who are allowed to view confidential issues. # This will still show a discrepancy on issues number but should be less than before. # Check https://gitlab.com/gitlab-org/gitlab-foss/issues/38418 description. # rubocop: disable CodeReuse/ActiveRecord - def self.query(projects, public_only: true) - issues_filtered_by_type = Issue.opened.with_issue_type(Issue::TYPES_FOR_LIST) - if public_only - issues_filtered_by_type.public_only.where(project: projects) + def self.query(projects, public_only: true, include_hidden: false) + if include_hidden + Issue.opened.with_issue_type(Issue::TYPES_FOR_LIST).where(project: projects) + elsif public_only + Issue.public_only.opened.with_issue_type(Issue::TYPES_FOR_LIST).where(project: projects) else - issues_filtered_by_type.where(project: projects) + Issue.without_hidden.opened.with_issue_type(Issue::TYPES_FOR_LIST).where(project: projects) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/services/projects/operations/update_service.rb b/app/services/projects/operations/update_service.rb index 51b8e3c6c54..ef74f3e6e7a 100644 --- a/app/services/projects/operations/update_service.rb +++ b/app/services/projects/operations/update_service.rb @@ -94,6 +94,7 @@ module Projects } } params[:error_tracking_setting_attributes][:token] = settings[:token] unless /\A\*+\z/.match?(settings[:token]) # Don't update token if we receive masked value + params[:error_tracking_setting_attributes][:integrated] = settings[:integrated] unless settings[:integrated].nil? params end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 074550e104d..27376173f07 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -20,8 +20,16 @@ module Projects raise TransferError, s_('TransferProject|Please select a new namespace for your project.') end - unless allowed_transfer?(current_user, project) - raise TransferError, s_('TransferProject|Transfer failed, please contact an admin.') + if @new_namespace.id == project.namespace_id + raise TransferError, s_('TransferProject|Project is already in this namespace.') + end + + unless allowed_transfer_project?(current_user, project) + raise TransferError, s_("TransferProject|You don't have permission to transfer this project.") + end + + unless allowed_to_transfer_to_namespace?(current_user, @new_namespace) + raise TransferError, s_("TransferProject|You don't have permission to transfer projects into that namespace.") end transfer(project) @@ -121,11 +129,12 @@ module Projects Milestones::TransferService.new(current_user, group, project).execute end - def allowed_transfer?(current_user, project) - @new_namespace && - can?(current_user, :change_namespace, project) && - @new_namespace.id != project.namespace_id && - current_user.can?(:transfer_projects, @new_namespace) + def allowed_transfer_project?(current_user, project) + current_user.can?(:change_namespace, project) + end + + def allowed_to_transfer_to_namespace?(current_user, namespace) + current_user.can?(:transfer_projects, namespace) end def update_namespace_and_visibility(to_namespace) diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index f96a6ee1255..dc75fe1014a 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -3,18 +3,9 @@ module Projects class UpdatePagesService < BaseService InvalidStateError = Class.new(StandardError) - FailedToExtractError = Class.new(StandardError) - ExclusiveLeaseTaken = Class.new(StandardError) - - include ::Pages::LegacyStorageLease - BLOCK_SIZE = 32.kilobytes PUBLIC_DIR = 'public' - # this has to be invalid group name, - # as it shares the namespace with groups - TMP_EXTRACT_PATH = '@pages.tmp' - # old deployment can be cached by pages daemon # so we need to give pages daemon some time update cache # 10 minutes is enough, but 30 feels safer @@ -42,7 +33,6 @@ module Projects validate_max_entries! build.artifacts_file.use_file do |artifacts_path| - deploy_to_legacy_storage(artifacts_path) create_pages_deployment(artifacts_path, build) success end @@ -78,70 +68,6 @@ module Projects ) end - def deploy_to_legacy_storage(artifacts_path) - # path today used by one project can later be used by another - # so we can't really scope this feature flag by project or group - return unless ::Settings.pages.local_store.enabled - - return if Feature.enabled?(:skip_pages_deploy_to_legacy_storage, project, default_enabled: :yaml) - - # Create temporary directory in which we will extract the artifacts - make_secure_tmp_dir(tmp_path) do |tmp_path| - extract_archive!(artifacts_path, tmp_path) - - # Check if we did extract public directory - archive_public_path = File.join(tmp_path, PUBLIC_DIR) - - raise InvalidStateError, 'pages miss the public folder' unless Dir.exist?(archive_public_path) - - validate_outdated_sha! - - deploy_page!(archive_public_path) - end - end - - def extract_archive!(artifacts_path, temp_path) - if artifacts.ends_with?('.zip') - extract_zip_archive!(artifacts_path, temp_path) - else - raise InvalidStateError, 'unsupported artifacts format' - end - end - - def extract_zip_archive!(artifacts_path, temp_path) - SafeZip::Extract.new(artifacts_path) - .extract(directories: [PUBLIC_DIR], to: temp_path) - rescue SafeZip::Extract::Error => e - raise FailedToExtractError, e.message - end - - def deploy_page!(archive_public_path) - deployed = try_obtain_lease do - deploy_page_unsafe!(archive_public_path) - true - end - - unless deployed - raise ExclusiveLeaseTaken, "Failed to deploy pages - other deployment is in progress" - end - end - - def deploy_page_unsafe!(archive_public_path) - # Do atomic move of pages - # Move and removal may not be atomic, but they are significantly faster then extracting and removal - # 1. We move deployed public to previous public path (file removal is slow) - # 2. We move temporary public to be deployed public - # 3. We remove previous public path - FileUtils.mkdir_p(pages_path) - begin - FileUtils.move(public_path, previous_public_path) - rescue StandardError - end - FileUtils.move(archive_public_path, public_path) - ensure - FileUtils.rm_r(previous_public_path, force: true) - end - def create_pages_deployment(artifacts_path, build) sha256 = build.job_artifacts_archive.file_sha256 @@ -165,22 +91,6 @@ module Projects ) end - def tmp_path - @tmp_path ||= File.join(::Settings.pages.path, TMP_EXTRACT_PATH) - end - - def pages_path - @pages_path ||= project.pages_path - end - - def public_path - @public_path ||= File.join(pages_path, PUBLIC_DIR) - end - - def previous_public_path - @previous_public_path ||= File.join(pages_path, "#{PUBLIC_DIR}.#{SecureRandom.hex}") - end - def ref build.ref end @@ -216,20 +126,6 @@ module Projects @pages_deployments_failed_total_counter ||= Gitlab::Metrics.counter(:pages_deployments_failed_total, "Counter of GitLab Pages deployments which failed") end - def make_secure_tmp_dir(tmp_path) - FileUtils.mkdir_p(tmp_path) - path = Dir.mktmpdir(tmp_dir_prefix, tmp_path) - begin - yield(path) - ensure - FileUtils.remove_entry_secure(path) - end - end - - def tmp_dir_prefix - "project-#{project.id}-build-#{build.id}-" - end - def validate_state! raise InvalidStateError, 'missing pages artifacts' unless build.artifacts? raise InvalidStateError, 'missing artifacts metadata' unless build.artifacts_metadata? diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index d6e7f165d72..b87564fcaef 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -105,6 +105,7 @@ module Projects end update_pages_config if changing_pages_related_config? + update_pending_builds if shared_runners_toggled? end def after_rename_service(project) @@ -178,6 +179,16 @@ module Projects params[:topic_list] ||= topic_list if topic_list end + + def update_pending_builds + update_params = { instance_runners_enabled: project.shared_runners_enabled } + + ::Ci::UpdatePendingBuildService.new(project, update_params).execute + end + + def shared_runners_toggled? + project.previous_changes.include?('shared_runners_enabled') + end end end |