From 48aff82709769b098321c738f3444b9bdaa694c6 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 21 Oct 2020 07:08:36 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-5-stable-ee --- app/services/projects/alerting/notify_service.rb | 75 ++++++++++++---------- .../container_repository/cleanup_tags_service.rb | 5 -- .../container_repository/delete_tags_service.rb | 4 +- .../projects/create_from_template_service.rb | 18 +++++- app/services/projects/create_service.rb | 22 +++---- .../projects/gitlab_projects_import_service.rb | 1 + app/services/projects/operations/update_service.rb | 10 +++ app/services/projects/overwrite_project_service.rb | 12 ++-- .../projects/prometheus/alerts/notify_service.rb | 14 +++- app/services/projects/transfer_service.rb | 4 ++ app/services/projects/update_pages_service.rb | 16 +++++ .../projects/update_remote_mirror_service.rb | 11 +++- 12 files changed, 125 insertions(+), 67 deletions(-) (limited to 'app/services/projects') diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb index bfce5f1ad63..affac45fc3d 100644 --- a/app/services/projects/alerting/notify_service.rb +++ b/app/services/projects/alerting/notify_service.rb @@ -7,43 +7,34 @@ module Projects include ::IncidentManagement::Settings def execute(token) + return bad_request unless valid_payload_size? return forbidden unless alerts_service_activated? return unauthorized unless valid_token?(token) - alert = process_alert + process_alert return bad_request unless alert.persisted? - process_incident_issues(alert) if process_issues? + process_incident_issues if process_issues? send_alert_email if send_email? ServiceResponse.success - rescue Gitlab::Alerting::NotificationPayloadParser::BadPayloadError - bad_request end private delegate :alerts_service, :alerts_service_activated?, to: :project - def am_alert_params - strong_memoize(:am_alert_params) do - Gitlab::AlertManagement::AlertParams.from_generic_alert(project: project, payload: params.to_h) - end - end - def process_alert - existing_alert = find_alert_by_fingerprint(am_alert_params[:fingerprint]) - - if existing_alert - process_existing_alert(existing_alert) + if alert.persisted? + process_existing_alert else create_alert end end - def process_existing_alert(alert) - if am_alert_params[:ended_at].present? - process_resolved_alert(alert) + def process_existing_alert + if incoming_payload.ends_at.present? + process_resolved_alert else alert.register_new_event! end @@ -51,10 +42,10 @@ module Projects alert end - def process_resolved_alert(alert) + def process_resolved_alert return unless auto_close_incident? - if alert.resolve(am_alert_params[:ended_at]) + if alert.resolve(incoming_payload.ends_at) close_issue(alert.issue) end @@ -72,20 +63,16 @@ module Projects end def create_alert - alert = AlertManagement::Alert.create(am_alert_params.except(:ended_at)) - alert.execute_services if alert.persisted? - SystemNoteService.create_new_alert(alert, 'Generic Alert Endpoint') - - alert - end - - def find_alert_by_fingerprint(fingerprint) - return unless fingerprint + return unless alert.save - AlertManagement::Alert.not_resolved.for_fingerprint(project, fingerprint).first + alert.execute_services + SystemNoteService.create_new_alert( + alert, + alert.monitoring_tool || 'Generic Alert Endpoint' + ) end - def process_incident_issues(alert) + def process_incident_issues return if alert.issue ::IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id) @@ -94,11 +81,33 @@ module Projects def send_alert_email notification_service .async - .prometheus_alerts_fired(project, [parsed_payload]) + .prometheus_alerts_fired(project, [alert.attributes]) + end + + def alert + strong_memoize(:alert) do + existing_alert || new_alert + end + end + + def existing_alert + return unless incoming_payload.gitlab_fingerprint + + AlertManagement::Alert.not_resolved.for_fingerprint(project, incoming_payload.gitlab_fingerprint).first + end + + def new_alert + AlertManagement::Alert.new(**incoming_payload.alert_params, ended_at: nil) + end + + def incoming_payload + strong_memoize(:incoming_payload) do + Gitlab::AlertManagement::Payload.parse(project, params.to_h) + end end - def parsed_payload - Gitlab::Alerting::NotificationPayloadParser.call(params.to_h, project) + def valid_payload_size? + Gitlab::Utils::DeepSize.new(params).valid? end def valid_token?(token) diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index 31500043544..b8047a1ad71 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -4,7 +4,6 @@ module Projects module ContainerRepository class CleanupTagsService < BaseService def execute(container_repository) - return error('feature disabled') unless can_use? return error('access denied') unless can_destroy? return error('invalid regex') unless valid_regex? @@ -74,10 +73,6 @@ module Projects can?(current_user, :destroy_container_image, project) end - def can_use? - Feature.enabled?(:container_registry_cleanup, project, default_enabled: true) - end - def valid_regex? %w(name_regex_delete name_regex name_regex_keep).each do |param_name| regex = params[param_name] diff --git a/app/services/projects/container_repository/delete_tags_service.rb b/app/services/projects/container_repository/delete_tags_service.rb index fa3de4ae2ac..9fc3ec0aafb 100644 --- a/app/services/projects/container_repository/delete_tags_service.rb +++ b/app/services/projects/container_repository/delete_tags_service.rb @@ -26,9 +26,7 @@ module Projects end def delete_service - fast_delete_enabled = Feature.enabled?(:container_registry_fast_tag_delete, default_enabled: true) - - if fast_delete_enabled && @container_repository.client.supports_tag_delete? + if @container_repository.client.supports_tag_delete? ::Projects::ContainerRepository::Gitlab::DeleteTagsService.new(@container_repository, @tag_names) else ::Projects::ContainerRepository::ThirdParty::DeleteTagsService.new(@container_repository, @tag_names) diff --git a/app/services/projects/create_from_template_service.rb b/app/services/projects/create_from_template_service.rb index a207fd2c574..45b52a1861c 100644 --- a/app/services/projects/create_from_template_service.rb +++ b/app/services/projects/create_from_template_service.rb @@ -14,10 +14,16 @@ module Projects def execute return project unless validate_template! - file = built_in_template&.file + file = built_in_template&.file || sample_data_template&.file override_params = params.dup - params[:file] = file + + if built_in_template + params[:file] = built_in_template.file + elsif sample_data_template + params[:file] = sample_data_template.file + params[:sample_data] = true + end GitlabProjectsImportService.new(current_user, params, override_params).execute ensure @@ -27,7 +33,7 @@ module Projects private def validate_template! - return true if built_in_template + return true if built_in_template || sample_data_template project.errors.add(:template_name, _("'%{template_name}' is unknown or invalid" % { template_name: template_name })) false @@ -39,6 +45,12 @@ module Projects end end + def sample_data_template + strong_memoize(:sample_data_template) do + Gitlab::SampleDataTemplate.find(template_name) + end + end + def project @project ||= ::Project.new(namespace_id: params[:namespace_id]) end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 68b40fdd8f1..8f18a23aa0f 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -19,6 +19,10 @@ module Projects @project = Project.new(params) + # If a project is newly created it should have shared runners settings + # based on its group having it enabled. This is like the "default value" + @project.shared_runners_enabled = false if !params.key?(:shared_runners_enabled) && @project.group && @project.group.shared_runners_setting != 'enabled' + # Make sure that the user is allowed to use the specified visibility level if project_visibility.restricted? deny_visibility_level(@project, project_visibility.visibility_level) @@ -143,7 +147,7 @@ module Projects def create_readme commit_attrs = { - branch_name: Gitlab::CurrentSettings.default_branch_name.presence || 'master', + branch_name: @project.default_branch || 'master', commit_message: 'Initial commit', file_path: 'README.md', file_content: "# #{@project.name}\n\n#{@project.description}" @@ -161,10 +165,9 @@ module Projects @project.create_or_update_import_data(data: @import_data[:data], credentials: @import_data[:credentials]) if @import_data if @project.save - unless @project.gitlab_project_import? - create_services_from_active_instances_or_templates(@project) - @project.create_labels - end + Service.create_from_active_default_integrations(@project, :project_id, with_templates: true) + + @project.create_labels unless @project.gitlab_project_import? unless @project.import? raise 'Failed to create repository' unless @project.create_repository @@ -228,15 +231,6 @@ module Projects private - # rubocop: disable CodeReuse/ActiveRecord - def create_services_from_active_instances_or_templates(project) - Service.active.where(instance: true).or(Service.active.where(template: true)).group_by(&:type).each do |type, records| - service = records.find(&:instance?) || records.find(&:template?) - Service.build_from_integration(project.id, service).save! - end - end - # rubocop: enable CodeReuse/ActiveRecord - def project_namespace @project_namespace ||= Namespace.find_by_id(@params[:namespace_id]) || current_user.namespace end diff --git a/app/services/projects/gitlab_projects_import_service.rb b/app/services/projects/gitlab_projects_import_service.rb index 2e192942b9c..27cce15f97d 100644 --- a/app/services/projects/gitlab_projects_import_service.rb +++ b/app/services/projects/gitlab_projects_import_service.rb @@ -66,6 +66,7 @@ module Projects end if template_file + data[:sample_data] = params.delete(:sample_data) if params.key?(:sample_data) params[:import_type] = 'gitlab_project' end diff --git a/app/services/projects/operations/update_service.rb b/app/services/projects/operations/update_service.rb index 7af489c3751..7dfe7fffa1b 100644 --- a/app/services/projects/operations/update_service.rb +++ b/app/services/projects/operations/update_service.rb @@ -18,6 +18,7 @@ module Projects .merge(grafana_integration_params) .merge(prometheus_integration_params) .merge(incident_management_setting_params) + .merge(tracing_setting_params) end def alerting_setting_params @@ -121,6 +122,15 @@ module Projects { incident_management_setting_attributes: attrs } end + + def tracing_setting_params + attr = params[:tracing_setting_attributes] + return {} unless attr + + destroy = attr[:external_url].blank? + + { tracing_setting_attributes: attr.merge(_destroy: destroy) } + end end end end diff --git a/app/services/projects/overwrite_project_service.rb b/app/services/projects/overwrite_project_service.rb index 958a00afbb8..e681b5643ee 100644 --- a/app/services/projects/overwrite_project_service.rb +++ b/app/services/projects/overwrite_project_service.rb @@ -32,12 +32,12 @@ module Projects def move_before_destroy_relationships(source_project) options = { remove_remaining_elements: false } - ::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, options) - ::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, options) - ::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, options) - ::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, options) - ::Projects::MoveForksService.new(@project, @current_user).execute(source_project, options) - ::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, options) + ::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, **options) + ::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, **options) + ::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, **options) + ::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, **options) + ::Projects::MoveForksService.new(@project, @current_user).execute(source_project, **options) + ::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, **options) add_source_project_to_fork_network(source_project) end diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index d32ead76d00..c002aca32db 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -125,7 +125,7 @@ module Projects notification_service .async - .prometheus_alerts_fired(project, firings) + .prometheus_alerts_fired(project, alerts_attributes) end def process_prometheus_alerts @@ -136,6 +136,18 @@ module Projects end end + def alerts_attributes + firings.map do |payload| + alert_params = Gitlab::AlertManagement::Payload.parse( + project, + payload, + monitoring_tool: Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus] + ).alert_params + + AlertManagement::Alert.new(alert_params).attributes + end + end + def bad_request ServiceResponse.error(message: 'Bad Request', http_status: :bad_request) end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index dba5177718d..013861631a1 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -88,6 +88,10 @@ module Projects # Move uploads move_project_uploads(project) + # If a project is being transferred to another group it means it can already + # have shared runners enabled but we need to check whether the new group allows that. + project.shared_runners_enabled = false if project.group && project.group.shared_runners_setting == 'disabled_and_unoverridable' + project.old_path_with_namespace = @old_path update_repository_configuration(@new_path) diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index ea37f2e4ec0..64b9eca9014 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -97,6 +97,7 @@ module Projects build.artifacts_file.use_file do |artifacts_path| SafeZip::Extract.new(artifacts_path) .extract(directories: [PUBLIC_DIR], to: temp_path) + create_pages_deployment(artifacts_path) end rescue SafeZip::Extract::Error => e raise FailedToExtractError, e.message @@ -118,6 +119,21 @@ module Projects FileUtils.rm_r(previous_public_path, force: true) end + def create_pages_deployment(artifacts_path) + return unless Feature.enabled?(:zip_pages_deployments, project) + + File.open(artifacts_path) do |file| + deployment = project.pages_deployments.create!(file: file) + project.pages_metadatum.update!(pages_deployment: deployment) + end + + # TODO: schedule old deployment removal https://gitlab.com/gitlab-org/gitlab/-/issues/235730 + rescue => e + # we don't want to break current pages deployment process if something goes wrong + # TODO: remove this rescue as part of https://gitlab.com/gitlab-org/gitlab/-/issues/245308 + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + def latest? # check if sha for the ref is still the most recent one # this helps in case when multiple deployments happens diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 5c41f00aac2..6115db54829 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -2,12 +2,14 @@ module Projects class UpdateRemoteMirrorService < BaseService + include Gitlab::Utils::StrongMemoize + MAX_TRIES = 3 def execute(remote_mirror, tries) return success unless remote_mirror.enabled? - if Gitlab::UrlBlocker.blocked_url?(CGI.unescape(Gitlab::UrlSanitizer.sanitize(remote_mirror.url))) + if Gitlab::UrlBlocker.blocked_url?(normalized_url(remote_mirror.url)) return error("The remote mirror URL is invalid.") end @@ -27,6 +29,12 @@ module Projects private + def normalized_url(url) + strong_memoize(:normalized_url) do + CGI.unescape(Gitlab::UrlSanitizer.sanitize(url)) + end + end + def update_mirror(remote_mirror) remote_mirror.update_start! remote_mirror.ensure_remote! @@ -47,7 +55,6 @@ module Projects end def send_lfs_objects!(remote_mirror) - return unless Feature.enabled?(:push_mirror_syncs_lfs, project) return unless project.lfs_enabled? # TODO: Support LFS sync over SSH -- cgit v1.2.1