diff options
Diffstat (limited to 'app/services/projects')
17 files changed, 183 insertions, 157 deletions
diff --git a/app/services/projects/after_rename_service.rb b/app/services/projects/after_rename_service.rb index 2a35a07d555..a2cdb87e631 100644 --- a/app/services/projects/after_rename_service.rb +++ b/app/services/projects/after_rename_service.rb @@ -96,9 +96,19 @@ module Projects .rename_project(path_before, project_path, namespace_full_path) end - Gitlab::PagesTransfer - .new - .rename_project(path_before, project_path, namespace_full_path) + if project.pages_deployed? + # Block will be evaluated in the context of project so we need + # to bind to a local variable to capture it, as the instance + # variable and method aren't available on Project + path_before_local = @path_before + + project.run_after_commit_or_now do + Gitlab::PagesTransfer + .new + .async + .rename_project(path_before_local, path, namespace.full_path) + end + end end def log_completion @@ -110,8 +120,7 @@ module Projects def migrate_to_hashed_storage? Gitlab::CurrentSettings.hashed_storage_enabled? && - project.storage_upgradable? && - Feature.disabled?(:skip_hashed_storage_upgrade) + project.storage_upgradable? end def send_move_instructions? diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb index f883c8c7bd8..bfce5f1ad63 100644 --- a/app/services/projects/alerting/notify_service.rb +++ b/app/services/projects/alerting/notify_service.rb @@ -42,12 +42,39 @@ module Projects end def process_existing_alert(alert) - alert.register_new_event! + if am_alert_params[:ended_at].present? + process_resolved_alert(alert) + else + alert.register_new_event! + end + + alert + end + + def process_resolved_alert(alert) + return unless auto_close_incident? + + if alert.resolve(am_alert_params[:ended_at]) + close_issue(alert.issue) + end + + alert + end + + def close_issue(issue) + return if issue.blank? || issue.closed? + + ::Issues::CloseService + .new(project, User.alert_bot) + .execute(issue, system_note: false) + + SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed? end def create_alert - alert = AlertManagement::Alert.create(am_alert_params) + alert = AlertManagement::Alert.create(am_alert_params.except(:ended_at)) alert.execute_services if alert.persisted? + SystemNoteService.create_new_alert(alert, 'Generic Alert Endpoint') alert end diff --git a/app/services/projects/container_repository/gitlab/delete_tags_service.rb b/app/services/projects/container_repository/gitlab/delete_tags_service.rb index 18049648e26..cee94b994a3 100644 --- a/app/services/projects/container_repository/gitlab/delete_tags_service.rb +++ b/app/services/projects/container_repository/gitlab/delete_tags_service.rb @@ -5,6 +5,11 @@ module Projects module Gitlab class DeleteTagsService include BaseServiceUtility + include ::Gitlab::Utils::StrongMemoize + + DISABLED_TIMEOUTS = [nil, 0].freeze + + TimeoutError = Class.new(StandardError) def initialize(container_repository, tag_names) @container_repository = container_repository @@ -17,12 +22,42 @@ module Projects def execute return success(deleted: []) if @tag_names.empty? + delete_tags + rescue TimeoutError => e + ::Gitlab::ErrorTracking.track_exception(e, tags_count: @tag_names&.size, container_repository_id: @container_repository&.id) + error('timeout while deleting tags') + end + + private + + def delete_tags + start_time = Time.zone.now + deleted_tags = @tag_names.select do |name| + raise TimeoutError if timeout?(start_time) + @container_repository.delete_tag_by_name(name) end deleted_tags.any? ? success(deleted: deleted_tags) : error('could not delete tags') end + + def timeout?(start_time) + return false unless throttling_enabled? + return false if service_timeout.in?(DISABLED_TIMEOUTS) + + (Time.zone.now - start_time) > service_timeout + end + + def throttling_enabled? + strong_memoize(:feature_flag) do + Feature.enabled?(:container_registry_expiration_policies_throttling) + end + end + + def service_timeout + ::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout + end end end end diff --git a/app/services/projects/container_repository/third_party/delete_tags_service.rb b/app/services/projects/container_repository/third_party/delete_tags_service.rb index 6504172109e..404642acf72 100644 --- a/app/services/projects/container_repository/third_party/delete_tags_service.rb +++ b/app/services/projects/container_repository/third_party/delete_tags_service.rb @@ -15,7 +15,7 @@ module Projects # This is a hack as the registry doesn't support deleting individual # tags. This code effectively pushes a dummy image and assigns the tag to it. # This way when the tag is deleted only the dummy image is affected. - # This is used to preverse compatibility with third-party registries that + # This is used to preserve compatibility with third-party registries that # don't support fast delete. # See https://gitlab.com/gitlab-org/gitlab/issues/15737 for a discussion def execute diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 33ed1151407..68b40fdd8f1 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -114,8 +114,13 @@ module Projects # completes), and any other affected users in the background def setup_authorizations if @project.group - current_user.project_authorizations.create!(project: @project, - access_level: @project.group.max_member_access_for_user(current_user)) + group_access_level = @project.group.max_member_access_for_user(current_user, + only_concrete_membership: true) + + if group_access_level > GroupMember::NO_ACCESS + current_user.project_authorizations.create!(project: @project, + access_level: group_access_level) + end if Feature.enabled?(:specialized_project_authorization_workers) AuthorizedProjectUpdate::ProjectCreateWorker.perform_async(@project.id) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 37487261f2c..bec75657530 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -28,7 +28,7 @@ module Projects Projects::UnlinkForkService.new(project, current_user).execute - attempt_destroy_transaction(project) + attempt_destroy(project) system_hook_service.execute_hooks_for(project, :destroy) log_info("Project \"#{project.full_path}\" was deleted") @@ -98,29 +98,35 @@ module Projects log_error("Deletion failed on #{project.full_path} with the following message: #{message}") end - def attempt_destroy_transaction(project) + def attempt_destroy(project) unless remove_registry_tags raise_error(s_('DeleteProject|Failed to remove some tags in project container registry. Please try again or contact administrator.')) end project.leave_pool_repository - Project.transaction do - log_destroy_event - trash_relation_repositories! - trash_project_repositories! - - # Rails attempts to load all related records into memory before - # destroying: https://github.com/rails/rails/issues/22510 - # This ensures we delete records in batches. - # - # Exclude container repositories because its before_destroy would be - # called multiple times, and it doesn't destroy any database records. - project.destroy_dependent_associations_in_batches(exclude: [:container_repositories, :snippets]) - project.destroy! + if Gitlab::Ci::Features.project_transactionless_destroy?(project) + destroy_project_related_records(project) + else + Project.transaction { destroy_project_related_records(project) } end end + def destroy_project_related_records(project) + log_destroy_event + trash_relation_repositories! + trash_project_repositories! + + # Rails attempts to load all related records into memory before + # destroying: https://github.com/rails/rails/issues/22510 + # This ensures we delete records in batches. + # + # Exclude container repositories because its before_destroy would be + # called multiple times, and it doesn't destroy any database records. + project.destroy_dependent_associations_in_batches(exclude: [:container_repositories, :snippets]) + project.destroy! + end + def log_destroy_event log_info("Attempting to destroy #{project.full_path} (#{project.id})") end diff --git a/app/services/projects/download_service.rb b/app/services/projects/download_service.rb index aba175eb79b..9810db84605 100644 --- a/app/services/projects/download_service.rb +++ b/app/services/projects/download_service.rb @@ -27,7 +27,7 @@ module Projects end def http?(url) - url =~ /\A#{URI.regexp(%w(http https))}\z/ + url =~ /\A#{URI::DEFAULT_PARSER.make_regexp(%w(http https))}\z/ end def valid_domain?(url) diff --git a/app/services/projects/fork_service.rb b/app/services/projects/fork_service.rb index bb660d47887..0b4963e356a 100644 --- a/app/services/projects/fork_service.rb +++ b/app/services/projects/fork_service.rb @@ -10,8 +10,8 @@ module Projects forked_project end - def valid_fork_targets - @valid_fork_targets ||= ForkTargetsFinder.new(@project, current_user).execute + def valid_fork_targets(options = {}) + @valid_fork_targets ||= ForkTargetsFinder.new(@project, current_user).execute(options) end def valid_fork_target?(namespace = target_namespace) diff --git a/app/services/projects/lfs_pointers/lfs_download_service.rb b/app/services/projects/lfs_pointers/lfs_download_service.rb index 52c73bcff03..d6e5b825e13 100644 --- a/app/services/projects/lfs_pointers/lfs_download_service.rb +++ b/app/services/projects/lfs_pointers/lfs_download_service.rb @@ -6,6 +6,9 @@ module Projects class LfsDownloadService < BaseService SizeError = Class.new(StandardError) OidError = Class.new(StandardError) + ResponseError = Class.new(StandardError) + + LARGE_FILE_SIZE = 1.megabytes attr_reader :lfs_download_object delegate :oid, :size, :credentials, :sanitized_url, to: :lfs_download_object, prefix: :lfs @@ -19,6 +22,7 @@ module Projects def execute return unless project&.lfs_enabled? && lfs_download_object return error("LFS file with oid #{lfs_oid} has invalid attributes") unless lfs_download_object.valid? + return link_existing_lfs_object! if Feature.enabled?(:lfs_link_existing_object, project, default_enabled: true) && lfs_size > LARGE_FILE_SIZE && lfs_object wrap_download_errors do download_lfs_file! @@ -29,7 +33,7 @@ module Projects def wrap_download_errors(&block) yield - rescue SizeError, OidError, StandardError => e + rescue SizeError, OidError, ResponseError, StandardError => e error("LFS file with oid #{lfs_oid} could't be downloaded from #{lfs_sanitized_url}: #{e.message}") end @@ -56,15 +60,13 @@ module Projects def download_and_save_file!(file) digester = Digest::SHA256.new - response = Gitlab::HTTP.get(lfs_sanitized_url, download_headers) do |fragment| + fetch_file do |fragment| digester << fragment file.write(fragment) raise_size_error! if file.size > lfs_size end - raise StandardError, "Received error code #{response.code}" unless response.success? - raise_size_error! if file.size != lfs_size raise_oid_error! if digester.hexdigest != lfs_oid end @@ -78,6 +80,12 @@ module Projects end end + def fetch_file(&block) + response = Gitlab::HTTP.get(lfs_sanitized_url, download_headers, &block) + + raise ResponseError, "Received error code #{response.code}" unless response.success? + end + def with_tmp_file create_tmp_storage_dir @@ -123,6 +131,29 @@ module Projects super end + + def lfs_object + @lfs_object ||= LfsObject.find_by_oid(lfs_oid) + end + + def link_existing_lfs_object! + existing_file = lfs_object.file.open + buffer_size = 0 + result = fetch_file do |fragment| + unless fragment == existing_file.read(fragment.size) + break error("LFS file with oid #{lfs_oid} cannot be linked with an existing LFS object") + end + + buffer_size += fragment.size + break success if buffer_size > LARGE_FILE_SIZE + end + + project.lfs_objects << lfs_object + + result + ensure + existing_file&.close + end end end end diff --git a/app/services/projects/open_issues_count_service.rb b/app/services/projects/open_issues_count_service.rb index 82632d63e5b..dc450311db2 100644 --- a/app/services/projects/open_issues_count_service.rb +++ b/app/services/projects/open_issues_count_service.rb @@ -68,10 +68,12 @@ module Projects # Check https://gitlab.com/gitlab-org/gitlab-foss/issues/38418 description. # rubocop: disable CodeReuse/ActiveRecord def self.query(projects, public_only: true) + issues_filtered_by_type = Issue.opened.with_issue_type(Issue::TYPES_FOR_LIST) + if public_only - Issue.opened.public_only.where(project: projects) + issues_filtered_by_type.public_only.where(project: projects) else - Issue.opened.where(project: projects) + issues_filtered_by_type.where(project: projects) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb deleted file mode 100644 index 54d09b354a1..00000000000 --- a/app/services/projects/propagate_service_template.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -module Projects - class PropagateServiceTemplate - BATCH_SIZE = 100 - - delegate :data_fields_present?, to: :template - - def self.propagate(template) - new(template).propagate - end - - def initialize(template) - @template = template - end - - def propagate - return unless template.active? - - propagate_projects_with_template - end - - private - - attr_reader :template - - def propagate_projects_with_template - loop do - batch = Project.uncached { Project.ids_without_integration(template, BATCH_SIZE) } - - bulk_create_from_template(batch) unless batch.empty? - - break if batch.size < BATCH_SIZE - end - end - - def bulk_create_from_template(batch) - service_list = ServiceList.new(batch, service_hash).to_array - - Project.transaction do - results = bulk_insert(*service_list) - - if data_fields_present? - data_list = DataList.new(results, data_fields_hash, template.data_fields.class).to_array - - bulk_insert(*data_list) - end - - run_callbacks(batch) - end - end - - def bulk_insert(klass, columns, values_array) - items_to_insert = values_array.map { |array| Hash[columns.zip(array)] } - - klass.insert_all(items_to_insert, returning: [:id]) - end - - def service_hash - @service_hash ||= template.to_service_hash - end - - def data_fields_hash - @data_fields_hash ||= template.to_data_fields_hash - end - - # rubocop: disable CodeReuse/ActiveRecord - def run_callbacks(batch) - if template.issue_tracker? - Project.where(id: batch).update_all(has_external_issue_tracker: true) - end - - if active_external_wiki? - Project.where(id: batch).update_all(has_external_wiki: true) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def active_external_wiki? - template.type == 'ExternalWikiService' - end - end -end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 0fb70feec86..dba5177718d 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -88,15 +88,14 @@ module Projects # Move uploads move_project_uploads(project) - # Move pages - Gitlab::PagesTransfer.new.move_project(project.path, @old_namespace.full_path, @new_namespace.full_path) - project.old_path_with_namespace = @old_path update_repository_configuration(@new_path) execute_system_hooks end + + move_pages(project) rescue Exception # rubocop:disable Lint/RescueException rollback_side_effects raise @@ -181,6 +180,13 @@ module Projects ) end + def move_pages(project) + return unless project.pages_deployed? + + transfer = Gitlab::PagesTransfer.new.async + transfer.move_project(project.path, @old_namespace.full_path, @new_namespace.full_path) + end + def old_wiki_repo_path "#{old_path}#{::Gitlab::GlRepository::WIKI.path_suffix}" end diff --git a/app/services/projects/unlink_fork_service.rb b/app/services/projects/unlink_fork_service.rb index b3cf27373cd..52aea8c51a5 100644 --- a/app/services/projects/unlink_fork_service.rb +++ b/app/services/projects/unlink_fork_service.rb @@ -2,21 +2,13 @@ module Projects class UnlinkForkService < BaseService - # If a fork is given, it: - # - # - Saves LFS objects to the root project - # - Close existing MRs coming from it - # - Is removed from the fork network - # - # If a root of fork(s) is given, it does the same, - # but not updating LFS objects (there'll be no related root to cache it). + # Close existing MRs coming from the project and remove it from the fork network def execute fork_network = @project.fork_network + forked_from = @project.forked_from_project return unless fork_network - save_lfs_objects - merge_requests = fork_network .merge_requests .opened @@ -41,7 +33,7 @@ module Projects # When the project getting out of the network is a node with parent # and children, both the parent and the node needs a cache refresh. - [@project.forked_from_project, @project].compact.each do |project| + [forked_from, @project].compact.each do |project| refresh_forks_count(project) end end @@ -51,22 +43,5 @@ module Projects def refresh_forks_count(project) Projects::ForksCountService.new(project).refresh_cache end - - # TODO: Remove this method once all LfsObjectsProject records are backfilled - # for forks. - # - # See https://gitlab.com/gitlab-org/gitlab/issues/122002 for more info. - def save_lfs_objects - return unless @project.forked? - - lfs_storage_project = @project.lfs_storage_project - - return unless lfs_storage_project - return if lfs_storage_project == @project # that project is being unlinked - - lfs_storage_project.lfs_objects.find_each do |lfs_object| - lfs_object.projects << @project unless lfs_object.projects.include?(@project) - end - end end end diff --git a/app/services/projects/update_pages_configuration_service.rb b/app/services/projects/update_pages_configuration_service.rb index 88c17d502df..67d388dc8a3 100644 --- a/app/services/projects/update_pages_configuration_service.rb +++ b/app/services/projects/update_pages_configuration_service.rb @@ -22,9 +22,6 @@ module Projects end success - rescue => e - Gitlab::ErrorTracking.track_exception(e) - error(e.message, pass_back: { exception: e }) end private diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index 334f5993d15..ea37f2e4ec0 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -52,7 +52,7 @@ module Projects def success @status.success - @project.mark_pages_as_deployed + @project.mark_pages_as_deployed(artifacts_archive: build.job_artifacts_archive) super end diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 7961f689259..5c41f00aac2 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -31,6 +31,9 @@ module Projects remote_mirror.update_start! remote_mirror.ensure_remote! + # LFS objects must be sent first, or the push has dangling pointers + send_lfs_objects!(remote_mirror) + response = remote_mirror.update_repository if response.divergent_refs.any? @@ -43,6 +46,23 @@ module Projects end end + def send_lfs_objects!(remote_mirror) + return unless Feature.enabled?(:push_mirror_syncs_lfs, project) + return unless project.lfs_enabled? + + # TODO: Support LFS sync over SSH + # https://gitlab.com/gitlab-org/gitlab/-/issues/249587 + return unless remote_mirror.url =~ /\Ahttps?:\/\//i + return unless remote_mirror.password_auth? + + Lfs::PushService.new( + project, + current_user, + url: remote_mirror.bare_url, + credentials: remote_mirror.credentials + ).execute + end + def retry_or_fail(mirror, message, tries) if tries < MAX_TRIES mirror.mark_for_retry!(message) diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index c9ba7cde199..bb430811497 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -144,11 +144,7 @@ module Projects def update_pages_config return unless project.pages_deployed? - if Feature.enabled?(:async_update_pages_config, project) - PagesUpdateConfigurationWorker.perform_async(project.id) - else - Projects::UpdatePagesConfigurationService.new(project).execute - end + PagesUpdateConfigurationWorker.perform_async(project.id) end def changing_pages_https_only? |