diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-18 08:17:02 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-18 08:17:02 +0000 |
| commit | b39512ed755239198a9c294b6a45e65c05900235 (patch) | |
| tree | d234a3efade1de67c46b9e5a38ce813627726aa7 /app/models | |
| parent | d31474cf3b17ece37939d20082b07f6657cc79a9 (diff) | |
| download | gitlab-ce-15.3.0-rc42.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-3-stable-eev15.3.0-rc42
Diffstat (limited to 'app/models')
143 files changed, 1665 insertions, 726 deletions
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 17b46f929c3..579f2c38ae6 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -790,10 +790,10 @@ class ApplicationSetting < ApplicationRecord def parsed_kroki_url @parsed_kroki_url ||= Gitlab::UrlBlocker.validate!(kroki_url, schemes: %w(http https), enforce_sanitization: true)[0] - rescue Gitlab::UrlBlocker::BlockedUrlError => error + rescue Gitlab::UrlBlocker::BlockedUrlError => e self.errors.add( :kroki_url, - "is not valid. #{error}" + "is not valid. #{e}" ) end diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index e9a0a156121..4d377855dea 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -122,7 +122,7 @@ module ApplicationSettingImplementation password_authentication_enabled_for_git: true, password_authentication_enabled_for_web: Settings.gitlab['signin_enabled'], performance_bar_allowed_group_id: nil, - personal_access_token_prefix: nil, + personal_access_token_prefix: 'glpat-', plantuml_enabled: false, plantuml_url: nil, polling_interval_multiplier: 1, diff --git a/app/models/approval.rb b/app/models/approval.rb index 899ea466315..9ded44fe425 100644 --- a/app/models/approval.rb +++ b/app/models/approval.rb @@ -2,11 +2,12 @@ class Approval < ApplicationRecord include CreatedAtFilterable + include Importable belongs_to :user belongs_to :merge_request - validates :merge_request_id, presence: true + validates :merge_request_id, presence: true, unless: :importing? validates :user_id, presence: true, uniqueness: { scope: [:merge_request_id] } scope :with_user, -> { joins(:user) } diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 8e8e9389e2d..0ad17cd8869 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -86,6 +86,18 @@ class AuditEvent < ApplicationRecord end end + def target_type + super || details[:target_type] + end + + def target_id + details[:target_id] + end + + def target_details + super || details[:target_details] + end + private def sanitize_message diff --git a/app/models/authentication_event.rb b/app/models/authentication_event.rb index 0ed197f32df..d5a5079acd6 100644 --- a/app/models/authentication_event.rb +++ b/app/models/authentication_event.rb @@ -20,7 +20,7 @@ class AuthenticationEvent < ApplicationRecord } scope :for_provider, ->(provider) { where(provider: provider) } - scope :ldap, -> { where('provider LIKE ?', 'ldap%')} + scope :ldap, -> { where('provider LIKE ?', 'ldap%') } def self.providers STATIC_PROVIDERS | Devise.omniauth_providers.map(&:to_s) diff --git a/app/models/blob.rb b/app/models/blob.rb index a12d856dc36..20d7c230aa2 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -93,8 +93,8 @@ class Blob < SimpleDelegator end def self.lazy(repository, commit_id, path, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) - BatchLoader.for([commit_id, path]).batch(key: repository) do |items, loader, args| - args[:key].blobs_at(items, blob_size_limit: blob_size_limit).each do |blob| + BatchLoader.for([commit_id, path]).batch(key: [:repository_blobs, repository]) do |items, loader, args| + args[:key].last.blobs_at(items, blob_size_limit: blob_size_limit).each do |blob| loader.call([blob.commit_id, blob.path], blob) if blob end end diff --git a/app/models/blob_viewer/metrics_dashboard_yml.rb b/app/models/blob_viewer/metrics_dashboard_yml.rb index 88643253d3d..cac6b2192d0 100644 --- a/app/models/blob_viewer/metrics_dashboard_yml.rb +++ b/app/models/blob_viewer/metrics_dashboard_yml.rb @@ -36,10 +36,10 @@ module BlobViewer yaml = ::Gitlab::Config::Loader::Yaml.new(blob.data).load_raw! ::PerformanceMonitoring::PrometheusDashboard.from_json(yaml) [] - rescue Gitlab::Config::Loader::FormatError => error - ["YAML syntax: #{error.message}"] - rescue ActiveModel::ValidationError => invalid - invalid.model.errors.messages.map { |messages| messages.join(': ') } + rescue Gitlab::Config::Loader::FormatError => e + ["YAML syntax: #{e.message}"] + rescue ActiveModel::ValidationError => e + e.model.errors.messages.map { |messages| messages.join(': ') } end def exhaustive_metrics_dashboard_validation @@ -47,8 +47,8 @@ module BlobViewer Gitlab::Metrics::Dashboard::Validator .errors(yaml, dashboard_path: blob.path, project: project) .map(&:message) - rescue Gitlab::Config::Loader::FormatError => error - [error.message] + rescue Gitlab::Config::Loader::FormatError => e + [e.message] end end end diff --git a/app/models/bulk_imports/configuration.rb b/app/models/bulk_imports/configuration.rb index 6d9f598583e..3b263ed0340 100644 --- a/app/models/bulk_imports/configuration.rb +++ b/app/models/bulk_imports/configuration.rb @@ -9,7 +9,7 @@ class BulkImports::Configuration < ApplicationRecord validates :url, :access_token, length: { maximum: 255 }, presence: true validates :url, public_url: { schemes: %w[http https], enforce_sanitization: true, ascii_only: true }, - allow_nil: true + allow_nil: true attr_encrypted :url, key: Settings.attr_encrypted_db_key_base_32, diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index cad2fafe640..e0a616b5fb4 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -52,9 +52,11 @@ class BulkImports::Entity < ApplicationRecord scope :by_user_id, ->(user_id) { joins(:bulk_import).where(bulk_imports: { user_id: user_id }) } scope :stale, -> { where('created_at < ?', 8.hours.ago).where(status: [0, 1]) } - scope :by_bulk_import_id, ->(bulk_import_id) { where(bulk_import_id: bulk_import_id)} + scope :by_bulk_import_id, ->(bulk_import_id) { where(bulk_import_id: bulk_import_id) } scope :order_by_created_at, -> (direction) { order(created_at: direction) } + alias_attribute :destination_slug, :destination_name + state_machine :status, initial: :created do state :created, value: 0 state :started, value: 1 diff --git a/app/models/chat_name.rb b/app/models/chat_name.rb index ff3f2663b73..60370c525d5 100644 --- a/app/models/chat_name.rb +++ b/app/models/chat_name.rb @@ -3,7 +3,7 @@ class ChatName < ApplicationRecord LAST_USED_AT_INTERVAL = 1.hour - belongs_to :integration, foreign_key: :service_id + belongs_to :integration belongs_to :user validates :user, presence: true @@ -11,8 +11,8 @@ class ChatName < ApplicationRecord validates :team_id, presence: true validates :chat_id, presence: true - validates :user_id, uniqueness: { scope: [:service_id] } - validates :chat_id, uniqueness: { scope: [:service_id, :team_id] } + validates :user_id, uniqueness: { scope: [:integration_id] } + validates :chat_id, uniqueness: { scope: [:integration_id, :team_id] } # Updates the "last_used_timestamp" but only if it wasn't already updated # recently. diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 13af5b1f8d1..3fda8693a58 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -19,7 +19,7 @@ module Ci belongs_to :project belongs_to :trigger_request has_many :sourced_pipelines, class_name: "::Ci::Sources::Pipeline", - foreign_key: :source_job_id + foreign_key: :source_job_id has_one :sourced_pipeline, class_name: "::Ci::Sources::Pipeline", foreign_key: :source_job_id has_one :downstream_pipeline, through: :sourced_pipeline, source: :pipeline @@ -114,7 +114,12 @@ module Ci def downstream_project_path strong_memoize(:downstream_project_path) do - options&.dig(:trigger, :project) + project = options&.dig(:trigger, :project) + next unless project + + scoped_variables.to_runner_variables.yield_self do |all_variables| + ::ExpandVariables.expand(project, all_variables) + end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 7f9697d0424..bf8817e6e78 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -194,7 +194,7 @@ module Ci after_save :stick_build_if_status_changed after_create unless: :importing? do |build| - run_after_commit { BuildHooksWorker.perform_async(build) } + run_after_commit { build.feature_flagged_execute_hooks } end class << self @@ -285,7 +285,7 @@ module Ci build.run_after_commit do BuildQueueWorker.perform_async(id) - BuildHooksWorker.perform_async(build) + build.feature_flagged_execute_hooks end end @@ -313,7 +313,7 @@ module Ci build.run_after_commit do build.ensure_persistent_ref - BuildHooksWorker.perform_async(build) + build.feature_flagged_execute_hooks end end @@ -322,6 +322,8 @@ module Ci build.run_status_commit_hooks! Ci::BuildFinishedWorker.perform_async(id) + + observe_report_types end end @@ -340,8 +342,8 @@ module Ci # rubocop: disable CodeReuse/ServiceClass Ci::RetryJobService.new(build.project, build.user).execute(build) # rubocop: enable CodeReuse/ServiceClass - rescue Gitlab::Access::AccessDeniedError => ex - Gitlab::AppLogger.error "Unable to auto-retry job #{build.id}: #{ex}" + rescue Gitlab::Access::AccessDeniedError => e + Gitlab::AppLogger.error "Unable to auto-retry job #{build.id}: #{e}" end end end @@ -490,11 +492,7 @@ module Ci if metadata&.expanded_environment_name.present? metadata.expanded_environment_name else - if ::Feature.enabled?(:ci_expand_environment_name_and_url, project) - ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) - else - ExpandVariables.expand(environment, -> { simple_variables }) - end + ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) end end end @@ -527,10 +525,14 @@ module Ci self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options end - def environment_deployment_tier + def environment_tier_from_options self.options.dig(:environment, :deployment_tier) if self.options end + def environment_tier + environment_tier_from_options || persisted_environment.try(:tier) + end + def triggered_by?(current_user) user == current_user end @@ -585,6 +587,7 @@ module Ci variables.concat(persisted_environment.predefined_variables) variables.append(key: 'CI_ENVIRONMENT_ACTION', value: environment_action) + variables.append(key: 'CI_ENVIRONMENT_TIER', value: environment_tier) # Here we're passing unexpanded environment_url for runner to expand, # and we need to make sure that CI_ENVIRONMENT_NAME and @@ -777,10 +780,20 @@ module Ci pending? && !any_runners_online? end + def feature_flagged_execute_hooks + if Feature.enabled?(:execute_build_hooks_inline, project) + execute_hooks + else + BuildHooksWorker.perform_async(self) + end + end + def execute_hooks return unless project return if user&.blocked? + ActiveRecord::Associations::Preloader.new.preload([self], { runner: :tags }) + project.execute_hooks(build_data.dup, :job_hooks) if project.has_active_hooks?(:job_hooks) project.execute_integrations(build_data.dup, :job_hooks) if project.has_active_integrations?(:job_hooks) end @@ -818,7 +831,11 @@ module Ci ) end - job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll + destroyed_artifacts = job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll + + Gitlab::Ci::Artifacts::Logger.log_deleted(destroyed_artifacts, 'Ci::Build#erase_erasable_artifacts!') + + destroyed_artifacts end def erase(opts = {}) @@ -831,7 +848,12 @@ module Ci ) end - job_artifacts.destroy_all # rubocop: disable Cop/DestroyAll + # TODO: We should use DestroyBatchService here + # See https://gitlab.com/gitlab-org/gitlab/-/issues/369132 + destroyed_artifacts = job_artifacts.destroy_all # rubocop: disable Cop/DestroyAll + + Gitlab::Ci::Artifacts::Logger.log_deleted(destroyed_artifacts, 'Ci::Build#erase') + erase_trace! update_erased!(opts[:erased_by]) end @@ -983,7 +1005,7 @@ module Ci def collect_test_reports!(test_reports) test_reports.get_suite(test_suite_name).tap do |test_suite| - each_report(Ci::JobArtifact::TEST_REPORT_FILE_TYPES) do |file_type, blob| + each_report(Ci::JobArtifact.file_types_for_report(:test)) do |file_type, blob| Gitlab::Ci::Parsers.fabricate!(file_type).parse!( blob, test_suite, @@ -994,7 +1016,7 @@ module Ci end def collect_accessibility_reports!(accessibility_report) - each_report(Ci::JobArtifact::ACCESSIBILITY_REPORT_FILE_TYPES) do |file_type, blob| + each_report(Ci::JobArtifact.file_types_for_report(:accessibility)) do |file_type, blob| Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, accessibility_report) end @@ -1002,7 +1024,7 @@ module Ci end def collect_codequality_reports!(codequality_report) - each_report(Ci::JobArtifact::CODEQUALITY_REPORT_FILE_TYPES) do |file_type, blob| + each_report(Ci::JobArtifact.file_types_for_report(:codequality)) do |file_type, blob| Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, codequality_report) end @@ -1010,7 +1032,7 @@ module Ci end def collect_terraform_reports!(terraform_reports) - each_report(::Ci::JobArtifact::TERRAFORM_REPORT_FILE_TYPES) do |file_type, blob, report_artifact| + each_report(::Ci::JobArtifact.file_types_for_report(:terraform)) do |file_type, blob, report_artifact| ::Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, terraform_reports, artifact: report_artifact) end @@ -1079,7 +1101,10 @@ module Ci end def drop_with_exit_code!(failure_reason, exit_code) - drop!(::Gitlab::Ci::Build::Status::Reason.new(self, failure_reason, exit_code)) + failure_reason ||= :unknown_failure + result = drop!(::Gitlab::Ci::Build::Status::Reason.new(self, failure_reason, exit_code)) + ::Ci::TrackFailedBuildWorker.perform_async(id, exit_code, failure_reason) + result end def exit_codes_defined? @@ -1149,6 +1174,21 @@ module Ci end end + def clone(current_user:, new_job_variables_attributes: []) + new_build = super + + if action? && new_job_variables_attributes.any? + new_build.job_variables = [] + new_build.job_variables_attributes = new_job_variables_attributes + end + + new_build + end + + def job_artifact_types + job_artifacts.map(&:file_type) + end + protected def run_status_commit_hooks! @@ -1256,6 +1296,20 @@ module Ci expires_in: RUNNERS_STATUS_CACHE_EXPIRATION ) { yield } end + + def observe_report_types + return unless ::Gitlab.com? && Feature.enabled?(:report_artifact_build_completed_metrics_on_build_completion) + + report_types = options&.dig(:artifacts, :reports)&.keys || [] + + report_types.each do |report_type| + next unless Ci::JobArtifact::REPORT_TYPES.include?(report_type) + + ::Gitlab::Ci::Artifacts::Metrics + .build_completed_report_type_counter(report_type) + .increment(status: status) + end + end end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 4ee661d89f4..5fc21ba3f28 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -19,6 +19,7 @@ module Ci before_create :set_build_project validates :build, presence: true + validates :id_tokens, json_schema: { filename: 'build_metadata_id_tokens' } validates :secrets, json_schema: { filename: 'build_metadata_secrets' } serialize :config_options, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 1ffa0e31f99..86de90983ff 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -39,8 +39,8 @@ module Ci def track_archival!(trace_artifact_id, checksum) update!(trace_artifact_id: trace_artifact_id, - checksum: checksum, - archived_at: Time.current) + checksum: checksum, + archived_at: Time.current) end def archival_attempts_message diff --git a/app/models/ci/deleted_object.rb b/app/models/ci/deleted_object.rb index aba7b73aba9..d36646aba66 100644 --- a/app/models/ci/deleted_object.rb +++ b/app/models/ci/deleted_object.rb @@ -27,8 +27,8 @@ module Ci def delete_file_from_storage file.remove! true - rescue StandardError => exception - Gitlab::ErrorTracking.track_exception(exception) + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e) false end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index ee7175a4f69..71d33f0bb63 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -13,14 +13,19 @@ module Ci include EachBatch include Gitlab::Utils::StrongMemoize - TEST_REPORT_FILE_TYPES = %w[junit].freeze - COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze - CODEQUALITY_REPORT_FILE_TYPES = %w[codequality].freeze - ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze NON_ERASABLE_FILE_TYPES = %w[trace].freeze - TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze - SAST_REPORT_TYPES = %w[sast].freeze - SECRET_DETECTION_REPORT_TYPES = %w[secret_detection].freeze + + REPORT_FILE_TYPES = { + sast: %w[sast], + secret_detection: %w[secret_detection], + test: %w[junit], + accessibility: %w[accessibility], + coverage: %w[cobertura], + codequality: %w[codequality], + terraform: %w[terraform], + sbom: %w[cyclonedx] + }.freeze + DEFAULT_FILE_NAMES = { archive: nil, metadata: nil, @@ -48,7 +53,8 @@ module Ci cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/361094 requirements: 'requirements.json', coverage_fuzzing: 'gl-coverage-fuzzing.json', - api_fuzzing: 'gl-api-fuzzing-report.json' + api_fuzzing: 'gl-api-fuzzing-report.json', + cyclonedx: 'gl-sbom.cdx.zip' }.freeze INTERNAL_TYPES = { @@ -88,7 +94,8 @@ module Ci terraform: :raw, requirements: :raw, coverage_fuzzing: :raw, - api_fuzzing: :raw + api_fuzzing: :raw, + cyclonedx: :zip }.freeze DOWNLOADABLE_TYPES = %w[ @@ -112,6 +119,7 @@ module Ci secret_detection requirements cluster_image_scanning + cyclonedx ].freeze TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze @@ -152,36 +160,14 @@ module Ci where(file_type: types) end - scope :all_reports, -> do - with_file_types(REPORT_TYPES.keys.map(&:to_s)) - end - - scope :sast_reports, -> do - with_file_types(SAST_REPORT_TYPES) - end - - scope :secret_detection_reports, -> do - with_file_types(SECRET_DETECTION_REPORT_TYPES) - end - - scope :test_reports, -> do - with_file_types(TEST_REPORT_FILE_TYPES) - end - - scope :accessibility_reports, -> do - with_file_types(ACCESSIBILITY_REPORT_FILE_TYPES) - end - - scope :coverage_reports, -> do - with_file_types(COVERAGE_REPORT_FILE_TYPES) - end - - scope :codequality_reports, -> do - with_file_types(CODEQUALITY_REPORT_FILE_TYPES) + REPORT_FILE_TYPES.each do |report_type, file_types| + scope "#{report_type}_reports", -> do + with_file_types(file_types) + end end - scope :terraform_reports, -> do - with_file_types(TERRAFORM_REPORT_FILE_TYPES) + scope :all_reports, -> do + with_file_types(REPORT_TYPES.keys.map(&:to_s)) end scope :erasable, -> do @@ -225,7 +211,8 @@ module Ci browser_performance: 24, ## EE-specific load_performance: 25, ## EE-specific api_fuzzing: 26, ## EE-specific - cluster_image_scanning: 27 ## EE-specific + cluster_image_scanning: 27, ## EE-specific + cyclonedx: 28 ## EE-specific } # `file_location` indicates where actual files are stored. @@ -259,6 +246,10 @@ module Ci end end + def self.file_types_for_report(report_type) + REPORT_FILE_TYPES.fetch(report_type) + end + def self.associated_file_types_for(file_type) return unless file_types.include?(file_type) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 95c6da4a7af..a94330270e2 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -52,15 +52,15 @@ module Ci belongs_to :ci_ref, class_name: 'Ci::Ref', foreign_key: :ci_ref_id, inverse_of: :pipelines has_internal_id :iid, scope: :project, presence: false, - track_if: -> { !importing? }, - ensure_if: -> { !importing? }, - init: ->(pipeline, scope) do - if pipeline - pipeline.project&.all_pipelines&.maximum(:iid) || pipeline.project&.all_pipelines&.count - elsif scope - ::Ci::Pipeline.where(**scope).maximum(:iid) - end - end + track_if: -> { !importing? }, + ensure_if: -> { !importing? }, + init: ->(pipeline, scope) do + if pipeline + pipeline.project&.all_pipelines&.maximum(:iid) || pipeline.project&.all_pipelines&.count + elsif scope + ::Ci::Pipeline.where(**scope).maximum(:iid) + end + end has_many :stages, -> { order(position: :asc) }, inverse_of: :pipeline has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline @@ -102,6 +102,7 @@ module Ci has_one :chat_data, class_name: 'Ci::PipelineChatData' has_many :triggered_pipelines, through: :sourced_pipelines, source: :pipeline + # Only includes direct and not nested children has_many :child_pipelines, -> { merge(Ci::Sources::Pipeline.same_project) }, through: :sourced_pipelines, source: :pipeline has_one :triggered_by_pipeline, through: :source_pipeline, source: :source_pipeline has_one :parent_pipeline, -> { merge(Ci::Sources::Pipeline.same_project) }, through: :source_pipeline, source: :source_pipeline @@ -389,7 +390,7 @@ module Ci end def self.latest_status(ref = nil) - newest_first(ref: ref).pluck(:status).first + newest_first(ref: ref).pick(:status) end def self.latest_successful_for_ref(ref) @@ -592,26 +593,20 @@ module Ci canceled? && auto_canceled_by_id? end - def cancel_running(retries: 1) - preloaded_relations = [:project, :pipeline, :deployment, :taggings] - - retry_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables| - cancelables.find_in_batches do |batch| - Preloaders::CommitStatusPreloader.new(batch).execute(preloaded_relations) - - batch.each do |job| - yield(job) if block_given? - job.cancel - end - end - end - end + # Cancel a pipelines cancelable jobs and optionally it's child pipelines cancelable jobs + # retries - # of times to retry if errors + # cascade_to_children - if true cancels all related child pipelines for parent child pipelines + # auto_canceled_by_pipeline_id - store the pipeline_id of the pipeline that triggered cancellation + # execute_async - if true cancel the children asyncronously + def cancel_running(retries: 1, cascade_to_children: true, auto_canceled_by_pipeline_id: nil, execute_async: true) + update(auto_canceled_by_id: auto_canceled_by_pipeline_id) if auto_canceled_by_pipeline_id - def auto_cancel_running(pipeline, retries: 1) - update(auto_canceled_by: pipeline) + cancel_jobs(cancelable_statuses, retries: retries, auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id) - cancel_running(retries: retries) do |job| - job.auto_canceled_by = pipeline + if cascade_to_children + # cancel any bridges that could spin up new child pipelines + cancel_jobs(bridges_in_self_and_descendants.cancelable, retries: retries, auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id) + cancel_children(auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id, execute_async: execute_async) end end @@ -953,6 +948,10 @@ module Ci Ci::Build.latest.where(pipeline: self_and_descendants) end + def bridges_in_self_and_descendants + Ci::Bridge.latest.where(pipeline: self_and_descendants) + end + def environments_in_self_and_descendants(deployment_status: nil) # We limit to 100 unique environments for application safety. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 @@ -986,6 +985,11 @@ module Ci object_hierarchy(project_condition: :same).base_and_descendants end + # With only parent-child pipelines + def all_child_pipelines + object_hierarchy(project_condition: :same).descendants + end + def self_and_descendants_complete? self_and_descendants.all?(&:complete?) end @@ -1152,6 +1156,10 @@ module Ci end end + def modified_paths_since(compare_to_sha) + project.repository.diff_stats(project.repository.merge_base(compare_to_sha, sha), sha).paths + end + def all_worktree_paths strong_memoize(:all_worktree_paths) do project.repository.ls_files(sha) @@ -1216,10 +1224,6 @@ module Ci stages.find_by(name: name) end - def find_stage_by_name!(name) - stages.find_by!(name: name) - end - def full_error_messages errors ? errors.full_messages.to_sentence : "" end @@ -1321,6 +1325,42 @@ module Ci private + def cancel_jobs(jobs, retries: 1, auto_canceled_by_pipeline_id: nil) + retry_lock(jobs, retries, name: 'ci_pipeline_cancel_running') do |statuses| + preloaded_relations = [:project, :pipeline, :deployment, :taggings] + + statuses.find_in_batches do |status_batch| + relation = CommitStatus.where(id: status_batch) + Preloaders::CommitStatusPreloader.new(relation).execute(preloaded_relations) + + relation.each do |job| + job.auto_canceled_by_id = auto_canceled_by_pipeline_id if auto_canceled_by_pipeline_id + job.cancel + end + end + end + end + + # For parent child-pipelines only (not multi-project) + def cancel_children(auto_canceled_by_pipeline_id: nil, execute_async: true) + all_child_pipelines.each do |child_pipeline| + if execute_async + ::Ci::CancelPipelineWorker.perform_async( + child_pipeline.id, + auto_canceled_by_pipeline_id + ) + else + child_pipeline.cancel_running( + # cascade_to_children is false because we iterate through children + # we also cancel bridges prior to prevent more children + cascade_to_children: false, + execute_async: execute_async, + auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id + ) + end + end + end + def add_message(severity, content) messages.build(severity: severity, content: content) end diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index f666629c8fd..a2ff49077be 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -101,7 +101,7 @@ module Ci :merge_train_pipeline?, to: :pipeline - def clone(current_user:) + def clone(current_user:, new_job_variables_attributes: []) new_attributes = self.class.clone_accessors.to_h do |attribute| [attribute, public_send(attribute)] # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index f41ad890184..6c3754d84d0 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -15,7 +15,7 @@ module Ci include Presentable include EachBatch - ignore_column :semver, remove_with: '15.3', remove_after: '2022-07-22' + ignore_column :semver, remove_with: '15.4', remove_after: '2022-08-22' add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration, expiration_enforced?: :token_expiration_enforced? @@ -437,7 +437,12 @@ module Ci cache_attributes(values) # We save data without validation, it will always change due to `contacted_at` - self.update_columns(values) if persist_cached_data? + if persist_cached_data? + version_updated = values.include?(:version) && values[:version] != version + + update_columns(values) + schedule_runner_version_update if version_updated + end end end @@ -477,7 +482,7 @@ module Ci private scope :with_upgrade_status, ->(upgrade_status) do - Ci::Runner.joins(:runner_version).where(runner_version: { status: upgrade_status }) + joins(:runner_version).where(runner_version: { status: upgrade_status }) end EXECUTOR_NAME_TO_TYPES = { @@ -565,6 +570,12 @@ module Ci errors.add(:runner, 'needs to be assigned to exactly one group') end end + + def schedule_runner_version_update + return unless version + + Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version) + end end end diff --git a/app/models/ci/runner_version.rb b/app/models/ci/runner_version.rb index 6b2d0060c9b..bbde98ee591 100644 --- a/app/models/ci/runner_version.rb +++ b/app/models/ci/runner_version.rb @@ -8,7 +8,6 @@ module Ci enum_with_nil status: { not_processed: nil, invalid_version: -1, - unknown: 0, not_available: 1, available: 2, recommended: 3 @@ -16,7 +15,6 @@ module Ci STATUS_DESCRIPTIONS = { invalid_version: 'Runner version is not valid.', - unknown: 'Upgrade status is unknown.', not_available: 'Upgrade is not available for the runner.', available: 'Upgrade is available for the runner.', recommended: 'Upgrade is available and recommended for the runner.' @@ -27,7 +25,7 @@ module Ci # This scope returns all versions that might need recalculating. For instance, once a version is considered # :recommended, it normally doesn't change status even if the instance is upgraded - scope :potentially_outdated, -> { where(status: [nil, :not_available, :available, :unknown]) } + scope :potentially_outdated, -> { where(status: [nil, :not_available, :available]) } validates :version, length: { maximum: 2048 } end diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 078b05ff779..9a35f1876c9 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -3,11 +3,8 @@ module Ci class SecureFile < Ci::ApplicationRecord include FileStoreMounter - include IgnorableColumns include Limitable - ignore_column :permissions, remove_with: '15.2', remove_after: '2022-06-22' - FILE_SIZE_LIMIT = 5.megabytes.freeze CHECKSUM_ALGORITHM = 'sha256' @@ -24,6 +21,7 @@ module Ci before_validation :assign_checksum scope :order_by_created_at, -> { order(created_at: :desc) } + scope :project_id_in, ->(ids) { where(project_id: ids) } default_value_for(:file_store) { Ci::SecureFileUploader.default_store } @@ -46,3 +44,5 @@ module Ci end end end + +Ci::SecureFile.prepend_mod diff --git a/app/models/commit.rb b/app/models/commit.rb index ca18cb50e02..bd60f02b532 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -190,7 +190,7 @@ class Commit def self.link_reference_pattern @link_reference_pattern ||= - super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})?(\.(?<extension>#{LINK_EXTENSION_PATTERN}))?/) + super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})?(\.(?<extension>#{LINK_EXTENSION_PATTERN}))?/o) end def to_reference(from = nil, full: false) diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 08f1eb3731e..e2f0de52bc9 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -50,7 +50,7 @@ class CommitRange end def self.link_reference_pattern - @link_reference_pattern ||= super("compare", /(?<commit_range>#{PATTERN})/) + @link_reference_pattern ||= super("compare", /(?<commit_range>#{PATTERN})/o) end # Initialize a CommitRange @@ -64,7 +64,7 @@ class CommitRange range_string = range_string.strip - unless range_string =~ /\A#{PATTERN}\z/ + unless range_string =~ /\A#{PATTERN}\z/o raise ArgumentError, "invalid CommitRange string format: #{range_string}" end diff --git a/app/models/commit_signatures/ssh_signature.rb b/app/models/commit_signatures/ssh_signature.rb index dbfbe0c3889..7a8d0653fcd 100644 --- a/app/models/commit_signatures/ssh_signature.rb +++ b/app/models/commit_signatures/ssh_signature.rb @@ -4,6 +4,6 @@ module CommitSignatures class SshSignature < ApplicationRecord include CommitSignature - belongs_to :key, optional: false + belongs_to :key, optional: true end end diff --git a/app/models/compare.rb b/app/models/compare.rb index 7f42e1ee491..f594a796987 100644 --- a/app/models/compare.rb +++ b/app/models/compare.rb @@ -92,7 +92,7 @@ class Compare def diff_refs Gitlab::Diff::DiffRefs.new( - base_sha: @straight ? start_commit_sha : base_commit_sha, + base_sha: @straight ? start_commit_sha : base_commit_sha, start_sha: start_commit_sha, head_sha: head_commit_sha ) diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index fb4ea4206f4..ee8e98ec1bf 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -5,11 +5,13 @@ module Ci extend ActiveSupport::Concern include ObjectStorable + include Gitlab::Ci::Artifacts::Logger STORE_COLUMN = :file_store NotSupportedAdapterError = Class.new(StandardError) FILE_FORMAT_ADAPTERS = { gzip: Gitlab::Ci::Build::Artifacts::Adapters::GzipStream, + zip: Gitlab::Ci::Build::Artifacts::Adapters::ZipStream, raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream }.freeze @@ -30,7 +32,7 @@ module Ci raise NotSupportedAdapterError, 'This file format requires a dedicated adapter' end - ::Gitlab::ApplicationContext.push(artifact: file.model) + log_artifacts_filesize(file.model) file.open do |stream| file_format_adapter_class.new(stream).each_blob(&blk) diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index 721cb14201f..910885c833f 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -17,8 +17,8 @@ module Ci ALIVE_STATUSES = (ACTIVE_STATUSES + ['created']).freeze CANCELABLE_STATUSES = (ALIVE_STATUSES + ['scheduled']).freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7, - scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze + failed: 4, canceled: 5, skipped: 6, manual: 7, + scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze UnknownStatusError = Class.new(StandardError) diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index aa9669ee208..8c3a05c23f0 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -20,6 +20,8 @@ module Ci delegate :interruptible, to: :metadata, prefix: false, allow_nil: true delegate :environment_auto_stop_in, to: :metadata, prefix: false, allow_nil: true delegate :set_cancel_gracefully, to: :metadata, prefix: false, allow_nil: false + delegate :id_tokens, to: :metadata, allow_nil: true + before_create :ensure_metadata end @@ -77,6 +79,14 @@ module Ci ensure_metadata.interruptible = value end + def id_tokens? + !!metadata&.id_tokens? + end + + def id_tokens=(value) + ensure_metadata.id_tokens = value + end + private def read_metadata_attribute(legacy_key, metadata_key, default_value = nil) diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index b41b1ba6008..65cf3246d11 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -82,18 +82,23 @@ module CounterAttribute lock_key = counter_lock_key(attribute) with_exclusive_lease(lock_key) do + previous_db_value = read_attribute(attribute) increment_key = counter_key(attribute) flushed_key = counter_flushed_key(attribute) increment_value = steal_increments(increment_key, flushed_key) + new_db_value = nil next if increment_value == 0 transaction do unsafe_update_counters(id, attribute => increment_value) redis_state { |redis| redis.del(flushed_key) } + new_db_value = reset.read_attribute(attribute) end execute_after_flush_callbacks + + log_flush_counter(attribute, increment_value, previous_db_value, new_db_value) end end @@ -115,15 +120,19 @@ module CounterAttribute def increment_counter(attribute, increment) if counter_attribute_enabled?(attribute) - redis_state do |redis| + new_value = redis_state do |redis| redis.incrby(counter_key(attribute), increment) end + + log_increment_counter(attribute, increment, new_value) end end def clear_counter!(attribute) if counter_attribute_enabled?(attribute) redis_state { |redis| redis.del(counter_key(attribute)) } + + log_clear_counter(attribute) end end @@ -184,4 +193,40 @@ module CounterAttribute rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError # a worker is already updating the counters end + + def log_increment_counter(attribute, increment, new_value) + payload = Gitlab::ApplicationContext.current.merge( + message: 'Increment counter attribute', + attribute: attribute, + project_id: project_id, + increment: increment, + new_counter_value: new_value, + current_db_value: read_attribute(attribute) + ) + + Gitlab::AppLogger.info(payload) + end + + def log_flush_counter(attribute, increment, previous_db_value, new_db_value) + payload = Gitlab::ApplicationContext.current.merge( + message: 'Flush counter attribute to database', + attribute: attribute, + project_id: project_id, + increment: increment, + previous_db_value: previous_db_value, + new_db_value: new_db_value + ) + + Gitlab::AppLogger.info(payload) + end + + def log_clear_counter(attribute) + payload = Gitlab::ApplicationContext.current.merge( + message: 'Clear counter attribute', + attribute: attribute, + project_id: project_id + ) + + Gitlab::AppLogger.info(payload) + end end diff --git a/app/models/concerns/cross_database_modification.rb b/app/models/concerns/cross_database_modification.rb index dea62f03f91..273d5f35e76 100644 --- a/app/models/concerns/cross_database_modification.rb +++ b/app/models/concerns/cross_database_modification.rb @@ -80,34 +80,22 @@ module CrossDatabaseModification end def transaction(**options, &block) - if track_gitlab_schema_in_current_transaction? - super(**options) do - # Hook into current transaction to ensure that once - # the `COMMIT` is executed the `gitlab_transactions_stack` - # will be allowing to execute `after_commit_queue` - record = TransactionStackTrackRecord.new(self, gitlab_schema) - - begin - connection.current_transaction.add_record(record) - - yield - ensure - record.done! - end + super(**options) do + # Hook into current transaction to ensure that once + # the `COMMIT` is executed the `gitlab_transactions_stack` + # will be allowing to execute `after_commit_queue` + record = TransactionStackTrackRecord.new(self, gitlab_schema) + + begin + connection.current_transaction.add_record(record) + + yield + ensure + record.done! end - else - super(**options, &block) end end - def track_gitlab_schema_in_current_transaction? - return false unless Feature::FlipperFeature.table_exists? - - Feature.enabled?(:track_gitlab_schema_in_current_transaction) - rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad - false - end - def gitlab_schema case self.name when 'ActiveRecord::Base', 'ApplicationRecord' diff --git a/app/models/concerns/database_event_tracking.rb b/app/models/concerns/database_event_tracking.rb new file mode 100644 index 00000000000..9f75b3ed4d8 --- /dev/null +++ b/app/models/concerns/database_event_tracking.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module DatabaseEventTracking + extend ActiveSupport::Concern + + included do + after_create_commit :publish_database_create_event + after_destroy_commit :publish_database_destroy_event + after_update_commit :publish_database_update_event + end + + def publish_database_create_event + publish_database_event('create') + end + + def publish_database_destroy_event + publish_database_event('destroy') + end + + def publish_database_update_event + publish_database_event('update') + end + + def publish_database_event(name) + return unless Feature.enabled?(:product_intelligence_database_event_tracking) + + # Gitlab::Tracking#event is triggering Snowplow event + # Snowplow events are sent with usage of + # https://snowplow.github.io/snowplow-ruby-tracker/SnowplowTracker/AsyncEmitter.html + # that reports data asynchronously and does not impact performance nor carries a risk of + # rollback in case of error + + Gitlab::Tracking.event( + self.class.to_s, + "database_event_#{name}", + label: self.class.table_name, + namespace: try(:group) || try(:namespace), + property: name, + **filtered_record_attributes + ) + rescue StandardError => err + # this rescue should be a dead code due to utilization of AsyncEmitter, however + # since this concern is expected to be included in every model, it is better to + # prevent against any unexpected outcome + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err) + end + + def filtered_record_attributes + attributes + .with_indifferent_access + .slice(*self.class::SNOWPLOW_ATTRIBUTES) + end +end diff --git a/app/models/concerns/diff_positionable_note.rb b/app/models/concerns/diff_positionable_note.rb index 051158e5de5..7a6076c7d2e 100644 --- a/app/models/concerns/diff_positionable_note.rb +++ b/app/models/concerns/diff_positionable_note.rb @@ -17,7 +17,11 @@ module DiffPositionableNote %i(original_position position change_position).each do |meth| define_method "#{meth}=" do |new_position| if new_position.is_a?(String) - new_position = Gitlab::Json.parse(new_position) rescue nil + new_position = begin + Gitlab::Json.parse(new_position) + rescue StandardError + nil + end end if new_position.is_a?(Hash) diff --git a/app/models/concerns/enums/data_visualization_palette.rb b/app/models/concerns/enums/data_visualization_palette.rb index 25002e64ba6..6e712e79915 100644 --- a/app/models/concerns/enums/data_visualization_palette.rb +++ b/app/models/concerns/enums/data_visualization_palette.rb @@ -16,17 +16,17 @@ module Enums def self.weights { - '50' => 0, - '100' => 1, - '200' => 2, - '300' => 3, - '400' => 4, - '500' => 5, - '600' => 6, - '700' => 7, - '800' => 8, - '900' => 9, - '950' => 10 + '50' => 0, + '100' => 1, + '200' => 2, + '300' => 3, + '400' => 4, + '500' => 5, + '600' => 6, + '700' => 7, + '800' => 8, + '900' => 9, + '950' => 10 } end end diff --git a/app/models/concerns/enums/sbom.rb b/app/models/concerns/enums/sbom.rb new file mode 100644 index 00000000000..518efa669ad --- /dev/null +++ b/app/models/concerns/enums/sbom.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Enums + class Sbom + COMPONENT_TYPES = { + library: 0 + }.with_indifferent_access.freeze + + def self.component_types + COMPONENT_TYPES + end + end +end diff --git a/app/models/concerns/expirable.rb b/app/models/concerns/expirable.rb index e029ada84f0..5975ea23723 100644 --- a/app/models/concerns/expirable.rb +++ b/app/models/concerns/expirable.rb @@ -6,7 +6,10 @@ module Expirable DAYS_TO_EXPIRE = 7 included do - scope :expired, -> { where('expires_at <= ?', Time.current) } + scope :not, ->(scope) { where(scope.arel.constraints.reduce(:and).not) } + + scope :expired, -> { where('expires_at IS NOT NULL AND expires_at <= ?', Time.current) } + scope :not_expired, -> { self.not(expired) } end def expired? diff --git a/app/models/concerns/featurable.rb b/app/models/concerns/featurable.rb index 08189d83534..3b741208221 100644 --- a/app/models/concerns/featurable.rb +++ b/app/models/concerns/featurable.rb @@ -30,9 +30,9 @@ module Featurable STRING_OPTIONS = HashWithIndifferentAccess.new({ 'disabled' => DISABLED, - 'private' => PRIVATE, - 'enabled' => ENABLED, - 'public' => PUBLIC + 'private' => PRIVATE, + 'enabled' => ENABLED, + 'public' => PUBLIC }).freeze class_methods do @@ -114,7 +114,7 @@ module Featurable self.errors.add(field, "cannot have public visibility level") if not_allowed end - (self.class.available_features - feature_validation_exclusion).each {|f| validator.call("#{f}_access_level")} + (self.class.available_features - feature_validation_exclusion).each { |f| validator.call("#{f}_access_level") } end # Features that we should exclude from the validation diff --git a/app/models/concerns/integrations/base_data_fields.rb b/app/models/concerns/integrations/base_data_fields.rb index 11bdd3aae7b..2870922d90d 100644 --- a/app/models/concerns/integrations/base_data_fields.rb +++ b/app/models/concerns/integrations/base_data_fields.rb @@ -4,15 +4,10 @@ module Integrations module BaseDataFields extend ActiveSupport::Concern - LEGACY_FOREIGN_KEY_NAME = %w( - Integrations::IssueTrackerData - Integrations::JiraTrackerData - ).freeze - included do # TODO: Once we rename the tables we can't rely on `table_name` anymore. # https://gitlab.com/gitlab-org/gitlab/-/issues/331953 - belongs_to :integration, inverse_of: self.table_name.to_sym, foreign_key: foreign_key_name + belongs_to :integration, inverse_of: self.table_name.to_sym, foreign_key: :integration_id validates :integration, presence: true end @@ -26,16 +21,6 @@ module Integrations algorithm: 'aes-256-gcm' } end - - private - - # Older data field models use the `service_id` foreign key for the - # integration association. - def foreign_key_name - return :service_id if self.name.in?(LEGACY_FOREIGN_KEY_NAME) - - :integration_id - end end def activated? diff --git a/app/models/concerns/integrations/has_data_fields.rb b/app/models/concerns/integrations/has_data_fields.rb index 635147a2f3c..2671df873aa 100644 --- a/app/models/concerns/integrations/has_data_fields.rb +++ b/app/models/concerns/integrations/has_data_fields.rb @@ -44,8 +44,8 @@ module Integrations end included do - has_one :issue_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::IssueTrackerData' - has_one :jira_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::JiraTrackerData' + has_one :issue_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :integration_id, class_name: 'Integrations::IssueTrackerData' + has_one :jira_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :integration_id, class_name: 'Integrations::JiraTrackerData' has_one :zentao_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :integration_id, class_name: 'Integrations::ZentaoTrackerData' def data_fields diff --git a/app/models/concerns/integrations/has_web_hook.rb b/app/models/concerns/integrations/has_web_hook.rb index bc28c32695c..e6ca6cc7938 100644 --- a/app/models/concerns/integrations/has_web_hook.rb +++ b/app/models/concerns/integrations/has_web_hook.rb @@ -6,6 +6,7 @@ module Integrations included do after_save :update_web_hook!, if: :activated? + has_one :service_hook, inverse_of: :integration, foreign_key: :service_id end # Return the URL to be used for the webhook. diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 4dca07132ef..b81a9b51e1c 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -515,11 +515,23 @@ module Issuable changes end + def hook_reviewer_changes(old_associations) + changes = {} + old_reviewers = old_associations.fetch(:reviewers, reviewers) + + if old_reviewers != reviewers + changes[:reviewers] = [old_reviewers.map(&:hook_attrs), reviewers.map(&:hook_attrs)] + end + + changes + end + def to_hook_data(user, old_associations: {}) changes = previous_changes if old_associations.present? changes.merge!(hook_association_changes(old_associations)) + changes.merge!(hook_reviewer_changes(old_associations)) if allows_reviewers? end Gitlab::DataBuilder::Issuable.new(self).build(user: user, changes: changes) @@ -537,6 +549,10 @@ module Issuable labels.map(&:hook_attrs) end + def allows_scoped_labels? + false + end + # Convert this Issuable class name to a format usable by Ability definitions # # Examples: @@ -550,7 +566,7 @@ module Issuable # Returns a Hash of attributes to be used for Twitter card metadata def card_attributes { - 'Author' => author.try(:name), + 'Author' => author.try(:name), 'Assignee' => assignee_list } end diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index f59b5d1ecc8..8130adf05f1 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -109,6 +109,7 @@ module Participable when User participants << source when Participable + next if skippable_system_notes?(source, participants) next unless !verify_access || source_visible_to_user?(source, current_user) source.class.participant_attrs.each do |attr| @@ -133,6 +134,13 @@ module Participable participants.merge(extractor.users) end + def skippable_system_notes?(source, participants) + source.is_a?(Note) && + source.system? && + source.author.in?(participants) && + !source.note.match?(User.reference_pattern) + end + def use_internal_notes_extractor_for?(source) source.is_a?(Note) && source.confidential? end diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index 900e8f7d39b..7613691bc2e 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -94,6 +94,18 @@ module ProjectFeaturesCompatibility write_feature_attribute_string(:container_registry_access_level, value) end + def environments_access_level=(value) + write_feature_attribute_string(:environments_access_level, value) + end + + def feature_flags_access_level=(value) + write_feature_attribute_string(:feature_flags_access_level, value) + end + + def releases_access_level=(value) + write_feature_attribute_string(:releases_access_level, value) + end + # TODO: Remove this method after we drop support for project create/edit APIs to set the # container_registry_enabled attribute. They can instead set the container_registry_access_level # attribute. diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index 86280097d19..df297017119 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -62,8 +62,8 @@ module PrometheusAdapter data: data, last_update: Time.current.utc } - rescue Gitlab::PrometheusClient::Error => err - { success: false, result: err.message } + rescue Gitlab::PrometheusClient::Error => e + { success: false, result: e.message } end def query_klass_for(query_name) diff --git a/app/models/concerns/repository_storage_movable.rb b/app/models/concerns/repository_storage_movable.rb index 1dd8eebeff3..b7fd52ab305 100644 --- a/app/models/concerns/repository_storage_movable.rb +++ b/app/models/concerns/repository_storage_movable.rb @@ -50,8 +50,8 @@ module RepositoryStorageMovable begin storage_move.container.set_repository_read_only!(skip_git_transfer_check: true) - rescue StandardError => err - storage_move.add_error(err.message) + rescue StandardError => e + storage_move.add_error(e.message) next false end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index 904c96b11b3..ee5774d4868 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -59,7 +59,7 @@ module Taskable end # Return a string that describes the current state of this Taskable's task - # list items, e.g. "12 of 20 tasks completed" + # list items, e.g. "12 of 20 checklist items completed" def task_status(short: false) return '' if description.blank? @@ -70,7 +70,7 @@ module Taskable end sum = tasks.summary - "#{sum.complete_count}#{prep}#{sum.item_count} #{'task'.pluralize(sum.item_count)}#{completed}" + "#{sum.complete_count}#{prep}#{sum.item_count} #{'checklist item'.pluralize(sum.item_count)}#{completed}" end # Return a short string that describes the current state of this Taskable's diff --git a/app/models/concerns/triggerable_hooks.rb b/app/models/concerns/triggerable_hooks.rb index 8fe34632430..e3800caa43f 100644 --- a/app/models/concerns/triggerable_hooks.rb +++ b/app/models/concerns/triggerable_hooks.rb @@ -2,22 +2,22 @@ module TriggerableHooks AVAILABLE_TRIGGERS = { - repository_update_hooks: :repository_update_events, - push_hooks: :push_events, - tag_push_hooks: :tag_push_events, - issue_hooks: :issues_events, - confidential_note_hooks: :confidential_note_events, + repository_update_hooks: :repository_update_events, + push_hooks: :push_events, + tag_push_hooks: :tag_push_events, + issue_hooks: :issues_events, + confidential_note_hooks: :confidential_note_events, confidential_issue_hooks: :confidential_issues_events, - note_hooks: :note_events, - merge_request_hooks: :merge_requests_events, - job_hooks: :job_events, - pipeline_hooks: :pipeline_events, - wiki_page_hooks: :wiki_page_events, - deployment_hooks: :deployment_events, - feature_flag_hooks: :feature_flag_events, - release_hooks: :releases_events, - member_hooks: :member_events, - subgroup_hooks: :subgroup_events + note_hooks: :note_events, + merge_request_hooks: :merge_requests_events, + job_hooks: :job_events, + pipeline_hooks: :pipeline_events, + wiki_page_hooks: :wiki_page_events, + deployment_hooks: :deployment_events, + feature_flag_hooks: :feature_flag_events, + release_hooks: :releases_events, + member_hooks: :member_events, + subgroup_hooks: :subgroup_events }.freeze extend ActiveSupport::Concern diff --git a/app/models/concerns/vulnerability_finding_helpers.rb b/app/models/concerns/vulnerability_finding_helpers.rb index 4cf36f83857..b5d48260072 100644 --- a/app/models/concerns/vulnerability_finding_helpers.rb +++ b/app/models/concerns/vulnerability_finding_helpers.rb @@ -50,7 +50,7 @@ module VulnerabilityFindingHelpers finding_data = report_finding.to_hash.except(:compare_key, :identifiers, :location, :scanner, :links, :signatures, :flags, :evidence) identifiers = report_finding.identifiers.map do |identifier| - Vulnerabilities::Identifier.new(identifier.to_hash) + Vulnerabilities::Identifier.new(identifier.to_hash.merge({ project: project })) end signatures = report_finding.signatures.map do |signature| Vulnerabilities::FindingSignature.new(signature.to_hash) @@ -72,6 +72,7 @@ module VulnerabilityFindingHelpers end finding.identifiers = identifiers + finding.primary_identifier = identifiers.first finding.signatures = signatures end end diff --git a/app/models/concerns/x509_serial_number_attribute.rb b/app/models/concerns/x509_serial_number_attribute.rb index e51ed95bf70..9dc53859ac0 100644 --- a/app/models/concerns/x509_serial_number_attribute.rb +++ b/app/models/concerns/x509_serial_number_attribute.rb @@ -33,8 +33,8 @@ module X509SerialNumberAttribute unless column.type == :binary raise ArgumentError, "x509_serial_number_attribute #{name.inspect} is invalid since the column type is not :binary" end - rescue StandardError => error - Gitlab::AppLogger.error "X509SerialNumberAttribute initialization: #{error.message}" + rescue StandardError => e + Gitlab::AppLogger.error "X509SerialNumberAttribute initialization: #{e.message}" raise end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index cdfd24e00aa..e10452c1081 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -19,6 +19,8 @@ class ContainerRepository < ApplicationRecord MIGRATION_PHASE_1_STARTED_AT = Date.new(2021, 11, 4).freeze MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze + MAX_TAGS_PAGES = 2000 + TooManyImportsError = Class.new(StandardError) belongs_to :project @@ -377,6 +379,10 @@ class ContainerRepository < ApplicationRecord migration_retries_count >= ContainerRegistry::Migration.max_retries - 1 end + def migrated? + MIGRATION_PHASE_1_ENDED_AT < self.created_at || import_done? + end + def last_import_step_done_at [migration_pre_import_done_at, migration_import_done_at, migration_aborted_at, migration_skipped_at].compact.max end @@ -427,6 +433,32 @@ class ContainerRepository < ApplicationRecord end end + def each_tags_page(page_size: 100, &block) + raise ArgumentError, 'not a migrated repository' unless migrated? + raise ArgumentError, 'block not given' unless block + + # dummy uri to initialize the loop + next_page_uri = URI('') + page_count = 0 + + while next_page_uri && page_count < MAX_TAGS_PAGES + last = Rack::Utils.parse_nested_query(next_page_uri.query)['last'] + current_page = gitlab_api_client.tags(self.path, page_size: page_size, last: last) + + if current_page&.key?(:response_body) + yield transform_tags_page(current_page[:response_body]) + next_page_uri = current_page.dig(:pagination, :next, :uri) + else + # no current page. Break the loop + next_page_uri = nil + end + + page_count += 1 + end + + raise 'too many pages requested' if page_count >= MAX_TAGS_PAGES + end + def tags_count return 0 unless manifest && manifest['tags'] @@ -550,7 +582,7 @@ class ContainerRepository < ApplicationRecord def self.find_by_path(path) self.find_by(project: path.repository_project, - name: path.repository_name) + name: path.repository_name) end private @@ -559,6 +591,16 @@ class ContainerRepository < ApplicationRecord self.migration_skipped_reason = reason finish_import end + + def transform_tags_page(tags_response_body) + return [] unless tags_response_body + + tags_response_body.map do |raw_tag| + tag = ContainerRegistry::Tag.new(self, raw_tag['name']) + tag.force_created_at_from_iso8601(raw_tag['created_at']) + tag + end + end end ContainerRepository.prepend_mod_with('ContainerRepository') diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index 09fbb93525b..625d68925c6 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -22,7 +22,7 @@ class CustomEmoji < ApplicationRecord presence: true, length: { maximum: 36 }, - format: { with: /\A#{NAME_REGEXP}\z/ } + format: { with: /\A#{NAME_REGEXP}\z/o } scope :by_name, -> (names) { where(name: names) } diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index 0f13c45b84d..f6455da890b 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -29,6 +29,12 @@ class CustomerRelations::Contact < ApplicationRecord validate :validate_email_format validate :validate_root_group + scope :order_scope_asc, ->(field) { order(arel_table[field].asc.nulls_last) } + scope :order_scope_desc, ->(field) { order(arel_table[field].desc.nulls_last) } + + scope :order_by_organization_asc, -> { includes(:organization).order("customer_relations_organizations.name ASC NULLS LAST") } + scope :order_by_organization_desc, -> { includes(:organization).order("customer_relations_organizations.name DESC NULLS LAST") } + def self.reference_prefix '[contact:' end @@ -56,6 +62,22 @@ class CustomerRelations::Contact < ApplicationRecord where(state: state) end + def self.sort_by_field(field, direction) + if direction == :asc + order_scope_asc(field) + else + order_scope_desc(field) + end + end + + def self.sort_by_organization(direction) + if direction == :asc + order_by_organization_asc + else + order_by_organization_desc + end + end + def self.sort_by_name order(Gitlab::Pagination::Keyset::Order.build([ Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( @@ -115,6 +137,10 @@ class CustomerRelations::Contact < ApplicationRecord where(group: group).update_all(group_id: group.root_ancestor.id) end + def self.counts_by_state + group(:state).count + end + private def validate_email_format diff --git a/app/models/customer_relations/contact_state_counts.rb b/app/models/customer_relations/contact_state_counts.rb new file mode 100644 index 00000000000..31c95e166bb --- /dev/null +++ b/app/models/customer_relations/contact_state_counts.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module CustomerRelations + class ContactStateCounts + include Gitlab::Utils::StrongMemoize + + attr_reader :group + + def self.declarative_policy_class + 'CustomerRelations::ContactPolicy' + end + + def initialize(current_user, group, params) + @current_user = current_user + @group = group + @params = params + end + + # Define method for each state + ::CustomerRelations::Contact.states.each_key do |state| + define_method(state) { counts[state] } + end + + def all + counts.values.sum + end + + private + + attr_reader :current_user, :params + + def counts + strong_memoize(:counts) do + Hash.new(0).merge(counts_by_state) + end + end + + def counts_by_state + ::Crm::ContactsFinder.counts_by_state(current_user, params.merge({ group: group })) + end + end +end diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 4ed38f578ee..94ac2405f61 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -40,6 +40,10 @@ class DeployKey < Key super || User.ghost end + def audit_details + title + end + def has_access_to?(project) deploy_keys_project_for(project).present? end @@ -62,4 +66,9 @@ class DeployKey < Key query end + + # This is used for the internal logic of AuditEvents::BuildService. + def impersonated? + false + end end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index c25ba6f9268..a3213a59bed 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -206,11 +206,6 @@ class Deployment < ApplicationRecord end end - def self.distinct_on_environment - order('environment_id, deployments.id DESC') - .select('DISTINCT ON (environment_id) deployments.*') - end - def self.find_successful_deployment!(iid) success.find_by!(iid: iid) end @@ -438,7 +433,7 @@ class Deployment < ApplicationRecord def tier_in_yaml return unless deployable - deployable.environment_deployment_tier + deployable.environment_tier_from_options end private diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb index feb1bf5438c..317399e780a 100644 --- a/app/models/design_management/design.rb +++ b/app/models/design_management/design.rb @@ -28,8 +28,8 @@ module DesignManagement has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_internal_id :iid, scope: :project, presence: true, - hook_names: %i[create update], # Deal with old records - track_if: -> { !importing? } + hook_names: %i[create update], # Deal with old records + track_if: -> { !importing? } validates :project, :filename, presence: true validates :issue, presence: true, unless: :importing? diff --git a/app/models/design_management/design_action.rb b/app/models/design_management/design_action.rb index 43dcce545d2..eae470a1ae2 100644 --- a/app/models/design_management/design_action.rb +++ b/app/models/design_management/design_action.rb @@ -21,7 +21,7 @@ module DesignManagement validates :action, presence: true, inclusion: { in: EVENT_FOR_GITALY_ACTION.keys } validates :content, absence: { if: :forbids_content?, - message: 'this action forbids content' }, + message: 'this action forbids content' }, presence: { if: :needs_content?, message: 'this action needs content' } diff --git a/app/models/environment.rb b/app/models/environment.rb index 68540ce0f5c..1950431446b 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -26,12 +26,11 @@ class Environment < ApplicationRecord has_many :self_managed_prometheus_alert_events, inverse_of: :environment has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment + # NOTE: If you preload multiple last deployments of environments, use Preloaders::Environments::DeploymentPreloader. has_one :last_deployment, -> { success.ordered }, class_name: 'Deployment', inverse_of: :environment - has_one :last_visible_deployment, -> { visible.distinct_on_environment }, inverse_of: :environment, class_name: 'Deployment' - has_one :last_visible_deployable, through: :last_visible_deployment, source: 'deployable', source_type: 'CommitStatus', disable_joins: true - has_one :last_visible_pipeline, through: :last_visible_deployable, source: 'pipeline', disable_joins: true + has_one :last_visible_deployment, -> { visible.order(id: :desc) }, inverse_of: :environment, class_name: 'Deployment' - has_one :upcoming_deployment, -> { upcoming.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment + has_one :upcoming_deployment, -> { upcoming.order(id: :desc) }, class_name: 'Deployment', inverse_of: :environment has_one :latest_opened_most_severe_alert, -> { order_severity_with_open_prometheus_alert }, class_name: 'AlertManagement::Alert', inverse_of: :environment before_validation :generate_slug, if: ->(env) { env.slug.blank? } @@ -56,8 +55,9 @@ class Environment < ApplicationRecord validates :external_url, length: { maximum: 255 }, - allow_nil: true, - addressable_url: true + allow_nil: true + + validate :safe_external_url delegate :manual_actions, :other_manual_actions, to: :last_deployment, allow_nil: true delegate :auto_rollback_enabled?, to: :project @@ -215,28 +215,11 @@ class Environment < ApplicationRecord deployable_id: last_deployment_pipeline.latest_builds.pluck(:id)) end - # NOTE: Below assocation overrides is a workaround for issue https://gitlab.com/gitlab-org/gitlab/-/issues/339908 - # It helps to avoid cross joins with the CI database. - # Caveat: It also overrides and losses the default AR caching mechanism. - # Read - https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68870#note_677227727 - - # NOTE: Association Preloads does not use the overriden definitions below. - # Association Preloads when preloading uses the original definitions from the relationships above. - # https://github.com/rails/rails/blob/75ac626c4e21129d8296d4206a1960563cc3d4aa/activerecord/lib/active_record/associations/preloader.rb#L158 - # But after preloading, when they are called it is using the overriden methods below. - # So we are checking for `association_cached?(:association_name)` in the overridden methods and calling `super` which inturn fetches the preloaded values. - - # Overriding association def last_visible_deployable - return super if association_cached?(:last_visible_deployable) - last_visible_deployment&.deployable end - # Overriding association def last_visible_pipeline - return super if association_cached?(:last_visible_pipeline) - last_visible_deployable&.pipeline end @@ -252,7 +235,6 @@ class Environment < ApplicationRecord Gitlab::Ci::Variables::Collection.new .append(key: 'CI_ENVIRONMENT_NAME', value: name) .append(key: 'CI_ENVIRONMENT_SLUG', value: slug) - .append(key: 'CI_ENVIRONMENT_TIER', value: tier) end def recently_updated_on_branch?(ref) @@ -329,11 +311,7 @@ class Environment < ApplicationRecord end def last_deployment_group - if ::Feature.enabled?(:batch_load_environment_last_deployment_group, project) - Deployment.last_deployment_group_for_environment(self) - else - legacy_last_deployment_group - end + Deployment.last_deployment_group_for_environment(self) end def reset_auto_stop @@ -493,6 +471,22 @@ class Environment < ApplicationRecord private + # We deliberately avoid using AddressableUrlValidator to allow users to update their environments even if they have + # misconfigured `environment:url` keyword. The external URL is presented as a clickable link on UI and not consumed + # in GitLab internally, thus we sanitize the URL before the persistence to make sure the rendered link is XSS safe. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/337417 + def safe_external_url + return unless self.external_url.present? + + new_external_url = Addressable::URI.parse(self.external_url) + + if Gitlab::Utils::SanitizeNodeLink::UNSAFE_PROTOCOLS.include?(new_external_url.normalized_scheme) + errors.add(:external_url, "#{new_external_url.normalized_scheme} scheme is not allowed") + end + rescue Addressable::URI::InvalidURIError + errors.add(:external_url, 'URI is invalid') + end + def rollout_status_available? has_terminals? end diff --git a/app/models/event.rb b/app/models/event.rb index 7760be3e817..a20ca0dc423 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -14,18 +14,18 @@ class Event < ApplicationRecord default_scope { reorder(nil) } # rubocop:disable Cop/DefaultScope ACTIONS = HashWithIndifferentAccess.new( - created: 1, - updated: 2, - closed: 3, - reopened: 4, - pushed: 5, - commented: 6, - merged: 7, - joined: 8, # User joined project - left: 9, # User left project - destroyed: 10, - expired: 11, # User left project due to expiry - approved: 12 + created: 1, + updated: 2, + closed: 3, + reopened: 4, + pushed: 5, + commented: 6, + merged: 7, + joined: 8, # User joined project + left: 9, # User left project + destroyed: 10, + expired: 11, # User left project due to expiry + approved: 12 ).freeze private_constant :ACTIONS @@ -36,15 +36,15 @@ class Event < ApplicationRecord ISSUE_ACTIONS = [:created, :updated, :closed, :reopened].freeze TARGET_TYPES = HashWithIndifferentAccess.new( - issue: Issue, - milestone: Milestone, - merge_request: MergeRequest, - note: Note, - project: Project, - snippet: Snippet, - user: User, - wiki: WikiPage::Meta, - design: DesignManagement::Design + issue: Issue, + milestone: Milestone, + merge_request: MergeRequest, + note: Note, + project: Project, + snippet: Snippet, + user: User, + wiki: WikiPage::Meta, + design: DesignManagement::Design ).freeze RESET_PROJECT_ACTIVITY_INTERVAL = 1.hour @@ -216,6 +216,10 @@ class Event < ApplicationRecord target_type == 'DesignManagement::Design' end + def work_item? + target_type == 'WorkItem' + end + def milestone target if milestone? end @@ -399,7 +403,8 @@ class Event < ApplicationRecord read_milestone: %i[milestone?], read_wiki: %i[wiki_page?], read_design: %i[design_note? design?], - read_note: %i[note?] + read_note: %i[note?], + read_work_item: %i[work_item?] } end diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb index a56e28859c9..2db074e733e 100644 --- a/app/models/gpg_key.rb +++ b/app/models/gpg_key.rb @@ -21,7 +21,7 @@ class GpgKey < ApplicationRecord presence: true, uniqueness: true, format: { - with: /\A#{KEY_PREFIX}((?!#{KEY_PREFIX})(?!#{KEY_SUFFIX}).)+#{KEY_SUFFIX}\Z/m, + with: /\A#{KEY_PREFIX}((?!#{KEY_PREFIX})(?!#{KEY_SUFFIX}).)+#{KEY_SUFFIX}\Z/mo, message: "is invalid. A valid public GPG key begins with '#{KEY_PREFIX}' and ends with '#{KEY_SUFFIX}'" } diff --git a/app/models/grafana_integration.rb b/app/models/grafana_integration.rb index 0358e37c58b..5cd5aa1b085 100644 --- a/app/models/grafana_integration.rb +++ b/app/models/grafana_integration.rb @@ -4,9 +4,9 @@ class GrafanaIntegration < ApplicationRecord belongs_to :project attr_encrypted :token, - mode: :per_attribute_iv, + mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32 + key: Settings.attr_encrypted_db_key_base_32 before_validation :check_token_changes diff --git a/app/models/group.rb b/app/models/group.rb index 6d8f8bd7613..55455d85531 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -149,7 +149,7 @@ class Group < Namespace add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption) ? :optional : :required }, - prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX after_create :post_create_hook after_destroy :post_destroy_hook @@ -176,6 +176,16 @@ class Group < Namespace .where(project_authorizations: { user_id: user_ids }) end + scope :project_creation_allowed, -> do + permitted_levels = [ + ::Gitlab::Access::DEVELOPER_MAINTAINER_PROJECT_ACCESS, + ::Gitlab::Access::MAINTAINER_PROJECT_ACCESS, + nil + ] + + where(project_creation_level: permitted_levels) + end + class << self def sort_by_attribute(method) if method == 'storage_size_desc' @@ -855,6 +865,14 @@ class Group < Namespace feature_flag_enabled_for_self_or_ancestor?(:work_items) end + def work_items_mvc_2_feature_flag_enabled? + feature_flag_enabled_for_self_or_ancestor?(:work_items_mvc_2) + end + + def work_items_create_from_markdown_feature_flag_enabled? + feature_flag_enabled_for_self_or_ancestor?(:work_items_create_from_markdown) + end + # Check for enabled features, similar to `Project#feature_available?` # NOTE: We still want to keep this after removing `Namespace#feature_available?`. override :feature_available? diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb index a70110c4076..8dd245a6ab5 100644 --- a/app/models/group_group_link.rb +++ b/app/models/group_group_link.rb @@ -14,6 +14,23 @@ class GroupGroupLink < ApplicationRecord presence: true scope :non_guests, -> { where('group_access > ?', Gitlab::Access::GUEST) } + + scope :with_owner_or_maintainer_access, -> do + where(group_access: [Gitlab::Access::OWNER, Gitlab::Access::MAINTAINER]) + end + + scope :groups_accessible_via, -> (shared_with_group_ids) do + links = where(shared_with_group_id: shared_with_group_ids) + # a group share also gives you access to the descendants of the group being shared, + # so we must include the descendants as well in the result. + Group.id_in(links.select(:shared_group_id)).self_and_descendants + end + + scope :groups_having_access_to, -> (shared_group_ids) do + links = where(shared_group_id: shared_group_ids) + Group.id_in(links.select(:shared_with_group_id)) + end + scope :preload_shared_with_groups, -> { preload(:shared_with_group) } scope :distinct_on_shared_with_group_id_with_group_access, -> do diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index f428d07cd7f..84ee23d77ce 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -12,14 +12,14 @@ class WebHook < ApplicationRecord BACKOFF_GROWTH_FACTOR = 2.0 attr_encrypted :token, - mode: :per_attribute_iv, + mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32 + key: Settings.attr_encrypted_db_key_base_32 attr_encrypted :url, - mode: :per_attribute_iv, + mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32 + key: Settings.attr_encrypted_db_key_base_32 attr_encrypted :url_variables, mode: :per_attribute_iv, @@ -57,14 +57,14 @@ class WebHook < ApplicationRecord !temporarily_disabled? && !permanently_disabled? end - def temporarily_disabled?(ignore_flag: false) - return false unless ignore_flag || web_hooks_disable_failed? + def temporarily_disabled? + return false unless web_hooks_disable_failed? disabled_until.present? && disabled_until >= Time.current end - def permanently_disabled?(ignore_flag: false) - return false unless ignore_flag || web_hooks_disable_failed? + def permanently_disabled? + return false unless web_hooks_disable_failed? recent_failures > FAILURE_THRESHOLD end @@ -126,13 +126,6 @@ class WebHook < ApplicationRecord save(validate: false) end - def active_state(ignore_flag: false) - return :permanently_disabled if permanently_disabled?(ignore_flag: ignore_flag) - return :temporarily_disabled if temporarily_disabled?(ignore_flag: ignore_flag) - - :enabled - end - # @return [Boolean] Whether or not the WebHook is currently throttled. def rate_limited? rate_limiter.rate_limited? diff --git a/app/models/integration.rb b/app/models/integration.rb index f5f701662e7..6d755016380 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -21,7 +21,7 @@ class Integration < ApplicationRecord asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat harbor irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email - pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao + pivotaltracker prometheus pumble pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao ].freeze # TODO Shimo is temporary disabled on group and instance-levels. @@ -48,6 +48,9 @@ class Integration < ApplicationRecord SECTION_TYPE_CONNECTION = 'connection' SECTION_TYPE_TRIGGER = 'trigger' + SNOWPLOW_EVENT_ACTION = 'perform_integrations_action' + SNOWPLOW_EVENT_LABEL = 'redis_hll_counters.ecosystem.ecosystem_total_unique_counts_monthly' + attr_encrypted :properties, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_32, @@ -89,7 +92,6 @@ class Integration < ApplicationRecord belongs_to :project, inverse_of: :integrations belongs_to :group, inverse_of: :integrations - has_one :service_hook, inverse_of: :integration, foreign_key: :service_id validates :project_id, presence: true, unless: -> { instance_level? || group_level? } validates :group_id, presence: true, unless: -> { instance_level? || project_level? } diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index 230dc6bb336..c3a4b84bb2d 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -63,11 +63,11 @@ module Integrations end def build_page(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:build_page] } + with_reactive_cache(sha, ref) { |cached| cached[:build_page] } end def commit_status(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } + with_reactive_cache(sha, ref) { |cached| cached[:commit_status] } end def execute(data) diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb index fe4a2f43b13..a4cec5f927b 100644 --- a/app/models/integrations/base_issue_tracker.rb +++ b/app/models/integrations/base_issue_tracker.rb @@ -100,8 +100,8 @@ module Integrations message = "#{self.type} received response #{response.code} when attempting to connect to #{self.project_url}" result = true end - rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Errno::ECONNRESET, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => error - message = "#{self.type} had an error when trying to connect to #{self.project_url}: #{error.message}" + rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Errno::ECONNRESET, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => e + message = "#{self.type} had an error when trying to connect to #{self.project_url}: #{e.message}" end log_info(message) result diff --git a/app/models/integrations/base_slash_commands.rb b/app/models/integrations/base_slash_commands.rb index a0ac5474893..e51d748b562 100644 --- a/app/models/integrations/base_slash_commands.rb +++ b/app/models/integrations/base_slash_commands.rb @@ -8,7 +8,7 @@ module Integrations prop_accessor :token - has_many :chat_names, foreign_key: :service_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :chat_names, foreign_key: :integration_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent def valid_token?(token) self.respond_to?(:token) && diff --git a/app/models/integrations/buildkite.rb b/app/models/integrations/buildkite.rb index def646c6d49..7a48e71b934 100644 --- a/app/models/integrations/buildkite.rb +++ b/app/models/integrations/buildkite.rb @@ -60,7 +60,7 @@ module Integrations end def commit_status(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } + with_reactive_cache(sha, ref) { |cached| cached[:commit_status] } end def commit_status_path(sha) diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb index 97e586c0662..bb0fb6b9079 100644 --- a/app/models/integrations/datadog.rb +++ b/app/models/integrations/datadog.rb @@ -15,75 +15,7 @@ module Integrations TAG_KEY_VALUE_RE = %r{\A [\w-]+ : .*\S.* \z}x.freeze - field :datadog_site, - placeholder: DEFAULT_DOMAIN, - help: -> do - ERB::Util.html_escape( - s_('DatadogIntegration|The Datadog site to send data to. To send data to the EU site, use %{codeOpen}datadoghq.eu%{codeClose}.') - ) % { - codeOpen: '<code>'.html_safe, - codeClose: '</code>'.html_safe - } - end - - field :api_url, - title: -> { s_('DatadogIntegration|API URL') }, - help: -> { s_('DatadogIntegration|(Advanced) The full URL for your Datadog site.') } - - field :api_key, - type: 'password', - title: -> { _('API key') }, - non_empty_password_title: -> { s_('ProjectService|Enter new API key') }, - non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current API key') }, - help: -> do - ERB::Util.html_escape( - s_('DatadogIntegration|%{linkOpen}API key%{linkClose} used for authentication with Datadog.') - ) % { - linkOpen: %Q{<a href="#{URL_API_KEYS_DOCS}" target="_blank" rel="noopener noreferrer">}.html_safe, - linkClose: '</a>'.html_safe - } - end, - required: true - - field :archive_trace_events, - type: 'checkbox', - title: -> { s_('Logs') }, - checkbox_label: -> { s_('Enable logs collection') }, - help: -> { s_('When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces.') } - - field :datadog_service, - title: -> { s_('DatadogIntegration|Service') }, - placeholder: 'gitlab-ci', - help: -> { s_('DatadogIntegration|Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments.') } - - field :datadog_env, - title: -> { s_('DatadogIntegration|Environment') }, - placeholder: 'ci', - help: -> do - ERB::Util.html_escape( - s_('DatadogIntegration|For self-managed deployments, set the %{codeOpen}env%{codeClose} tag for all the data sent to Datadog. %{linkOpen}How do I use tags?%{linkClose}') - ) % { - codeOpen: '<code>'.html_safe, - codeClose: '</code>'.html_safe, - linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, - linkClose: '</a>'.html_safe - } - end - - field :datadog_tags, - type: 'textarea', - title: -> { s_('DatadogIntegration|Tags') }, - placeholder: "tag:value\nanother_tag:value", - help: -> do - ERB::Util.html_escape( - s_('DatadogIntegration|Custom tags in Datadog. Enter one tag per line in the %{codeOpen}key:value%{codeClose} format. %{linkOpen}How do I use tags?%{linkClose}') - ) % { - codeOpen: '<code>'.html_safe, - codeClose: '</code>'.html_safe, - linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, - linkClose: '</a>'.html_safe - } - end + prop_accessor :datadog_site, :api_url, :api_key, :datadog_service, :datadog_env, :datadog_tags before_validation :strip_properties @@ -145,11 +77,92 @@ module Integrations end def fields + f = [ + { + type: 'text', + name: 'datadog_site', + placeholder: DEFAULT_DOMAIN, + help: ERB::Util.html_escape( + s_('DatadogIntegration|The Datadog site to send data to. To send data to the EU site, use %{codeOpen}datadoghq.eu%{codeClose}.') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe + }, + required: false + }, + { + type: 'text', + name: 'api_url', + title: s_('DatadogIntegration|API URL'), + help: s_('DatadogIntegration|(Advanced) The full URL for your Datadog site.'), + required: false + }, + { + type: 'password', + name: 'api_key', + title: _('API key'), + non_empty_password_title: s_('ProjectService|Enter new API key'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current API key'), + help: ERB::Util.html_escape( + s_('DatadogIntegration|%{linkOpen}API key%{linkClose} used for authentication with Datadog.') + ) % { + linkOpen: %Q{<a href="#{URL_API_KEYS_DOCS}" target="_blank" rel="noopener noreferrer">}.html_safe, + linkClose: '</a>'.html_safe + }, + required: true + } + ] + if Feature.enabled?(:datadog_integration_logs_collection, parent) - super - else - super.reject { _1.name == 'archive_trace_events' } + f.append({ + type: 'checkbox', + name: 'archive_trace_events', + title: s_('Logs'), + checkbox_label: s_('Enable logs collection'), + help: s_('When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces.'), + required: false + }) end + + f += [ + { + type: 'text', + name: 'datadog_service', + title: s_('DatadogIntegration|Service'), + placeholder: 'gitlab-ci', + help: s_('DatadogIntegration|Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments.') + }, + { + type: 'text', + name: 'datadog_env', + title: s_('DatadogIntegration|Environment'), + placeholder: 'ci', + help: ERB::Util.html_escape( + s_('DatadogIntegration|For self-managed deployments, set the %{codeOpen}env%{codeClose} tag for all the data sent to Datadog. %{linkOpen}How do I use tags?%{linkClose}') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe, + linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, + linkClose: '</a>'.html_safe + } + }, + { + type: 'textarea', + name: 'datadog_tags', + title: s_('DatadogIntegration|Tags'), + placeholder: "tag:value\nanother_tag:value", + help: ERB::Util.html_escape( + s_('DatadogIntegration|Custom tags in Datadog. Enter one tag per line in the %{codeOpen}key:value%{codeClose} format. %{linkOpen}How do I use tags?%{linkClose}') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe, + linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, + linkClose: '</a>'.html_safe + } + } + ] + + f end override :hook_url diff --git a/app/models/integrations/discord.rb b/app/models/integrations/discord.rb index ecabf23c90b..ec8a12e4760 100644 --- a/app/models/integrations/discord.rb +++ b/app/models/integrations/discord.rb @@ -33,10 +33,21 @@ module Integrations def default_fields [ - { type: "text", name: "webhook", placeholder: "https://discordapp.com/api/webhooks/…", help: "URL to the webhook for the Discord channel." }, - { type: "checkbox", name: "notify_only_broken_pipelines" }, + { + type: 'text', + section: SECTION_TYPE_CONNECTION, + name: 'webhook', + placeholder: 'https://discordapp.com/api/webhooks/…', + help: 'URL to the webhook for the Discord channel.' + }, + { + type: 'checkbox', + section: SECTION_TYPE_CONFIGURATION, + name: 'notify_only_broken_pipelines' + }, { type: 'select', + section: SECTION_TYPE_CONFIGURATION, name: 'branches_to_be_notified', title: s_('Integrations|Branches for which notifications are to be sent'), choices: self.class.branch_choices @@ -44,6 +55,26 @@ module Integrations ] end + def sections + [ + { + type: SECTION_TYPE_CONNECTION, + title: s_('Integrations|Connection details'), + description: help + }, + { + type: SECTION_TYPE_TRIGGER, + title: s_('Integrations|Trigger'), + description: s_('Integrations|An event will be triggered when one of the following items happen.') + }, + { + type: SECTION_TYPE_CONFIGURATION, + title: s_('Integrations|Notification settings'), + description: s_('Integrations|Configure the scope of notifications.') + } + ] + end + private def notify(message, opts) @@ -57,8 +88,8 @@ module Integrations embed.timestamp = Time.now.utc end end - rescue RestClient::Exception => error - log_error(error.message) + rescue RestClient::Exception => e + log_error(e.message) false end diff --git a/app/models/integrations/emails_on_push.rb b/app/models/integrations/emails_on_push.rb index ed12a3a8d63..25bda8c2bf0 100644 --- a/app/models/integrations/emails_on_push.rb +++ b/app/models/integrations/emails_on_push.rb @@ -71,7 +71,7 @@ module Integrations recipients, push_data, send_from_committer_email: send_from_committer_email?, - disable_diffs: disable_diffs? + disable_diffs: disable_diffs? ) end diff --git a/app/models/integrations/external_wiki.rb b/app/models/integrations/external_wiki.rb index bc2ea193a84..75fe6b6f164 100644 --- a/app/models/integrations/external_wiki.rb +++ b/app/models/integrations/external_wiki.rb @@ -5,6 +5,7 @@ module Integrations validates :external_wiki_url, presence: true, public_url: true, if: :activated? field :external_wiki_url, + section: SECTION_TYPE_CONNECTION, title: -> { s_('ExternalWikiService|External wiki URL') }, placeholder: -> { s_('ExternalWikiService|https://example.com/xxx/wiki/...') }, help: -> { s_('ExternalWikiService|Enter the URL to the external wiki.') }, @@ -28,6 +29,16 @@ module Integrations s_('Link an external wiki from the project\'s sidebar. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end + def sections + [ + { + type: SECTION_TYPE_CONNECTION, + title: s_('Integrations|Connection details'), + description: help + } + ] + end + def execute(_data) response = Gitlab::HTTP.get(properties['external_wiki_url'], verify: true) response.body if response.code == 200 diff --git a/app/models/integrations/harbor.rb b/app/models/integrations/harbor.rb index 82981493822..03913a71d47 100644 --- a/app/models/integrations/harbor.rb +++ b/app/models/integrations/harbor.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +require 'uri' module Integrations class Harbor < Integration @@ -20,7 +21,7 @@ module Integrations end def help - s_("HarborIntegration|After the Harbor integration is activated, global variables ‘$HARBOR_USERNAME’, ‘$HARBOR_PASSWORD’, ‘$HARBOR_URL’ and ‘$HARBOR_PROJECT’ will be created for CI/CD use.") + s_("HarborIntegration|After the Harbor integration is activated, global variables '$HARBOR_USERNAME', '$HARBOR_HOST', '$HARBOR_OCI', '$HARBOR_PASSWORD', '$HARBOR_URL' and '$HARBOR_PROJECT' will be created for CI/CD use.") end class << self @@ -78,8 +79,12 @@ module Integrations def ci_variables return [] unless activated? + oci_uri = URI.parse(url) + oci_uri.scheme = 'oci' [ { key: 'HARBOR_URL', value: url }, + { key: 'HARBOR_HOST', value: oci_uri.host }, + { key: 'HARBOR_OCI', value: oci_uri.to_s }, { key: 'HARBOR_PROJECT', value: project_name }, { key: 'HARBOR_USERNAME', value: username.gsub(/^robot\$/, 'robot$$') }, { key: 'HARBOR_PASSWORD', value: password, public: false, masked: true } diff --git a/app/models/integrations/jenkins.rb b/app/models/integrations/jenkins.rb index ab39d1f7b77..c68b5fd2a96 100644 --- a/app/models/integrations/jenkins.rb +++ b/app/models/integrations/jenkins.rb @@ -53,8 +53,8 @@ module Integrations begin result = execute(data) return { success: false, result: result[:message] } if result[:http_status] != 200 - rescue StandardError => error - return { success: false, result: error } + rescue StandardError => e + return { success: false, result: e } end { success: true, result: result[:message] } diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 566bbc456f8..3ca514ab1fd 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -18,6 +18,8 @@ module Integrations SECTION_TYPE_JIRA_TRIGGER = 'jira_trigger' SECTION_TYPE_JIRA_ISSUES = 'jira_issues' + SNOWPLOW_EVENT_CATEGORY = self.name + validates :url, public_url: true, presence: true, if: :activated? validates :api_url, public_url: true, allow_blank: true validates :username, presence: true, if: :activated? @@ -362,13 +364,17 @@ module Integrations ) true - rescue StandardError => error - log_exception(error, message: 'Issue transition failed', client_url: client_url) + rescue StandardError => e + log_exception(e, message: 'Issue transition failed', client_url: client_url) false end def transition_issue_to_done(issue) - transitions = issue.transitions rescue [] + transitions = begin + issue.transitions + rescue StandardError + [] + end transition = transitions.find do |transition| status = transition&.to&.statusCategory @@ -384,6 +390,22 @@ module Integrations key = "i_ecosystem_jira_service_#{action}" Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user.id) + + return unless Feature.enabled?(:route_hll_to_snowplow_phase2) + + optional_arguments = { + project: project, + namespace: group || project&.namespace + }.compact + + Gitlab::Tracking.event( + SNOWPLOW_EVENT_CATEGORY, + Integration::SNOWPLOW_EVENT_ACTION, + label: Integration::SNOWPLOW_EVENT_LABEL, + property: key, + user: user, + **optional_arguments + ) end def add_issue_solved_comment(issue, commit_id, commit_url) @@ -505,7 +527,7 @@ module Integrations self.project, entity_type.to_sym ], - id: entity_id, + id: entity_id, host: Settings.gitlab.base_url ) end @@ -538,9 +560,9 @@ module Integrations # Handle errors when doing Jira API calls def jira_request yield - rescue StandardError => error - @error = error - log_exception(error, message: 'Error sending message', client_url: client_url) + rescue StandardError => e + @error = e + log_exception(e, message: 'Error sending message', client_url: client_url) nil end diff --git a/app/models/integrations/packagist.rb b/app/models/integrations/packagist.rb index fda4822c19f..f91404dab23 100644 --- a/app/models/integrations/packagist.rb +++ b/app/models/integrations/packagist.rb @@ -6,14 +6,14 @@ module Integrations extend Gitlab::Utils::Override field :username, - title: -> { _('Username') }, + title: -> { s_('Username') }, help: -> { s_('Enter your Packagist username.') }, placeholder: '', required: true field :token, type: 'password', - title: -> { _('Token') }, + title: -> { s_('Token') }, help: -> { s_('Enter your Packagist token.') }, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, @@ -21,10 +21,11 @@ module Integrations required: true field :server, - title: -> { _('Server (optional)') }, + title: -> { s_('Server (optional)') }, help: -> { s_('Enter your Packagist server. Defaults to https://packagist.org.') }, placeholder: 'https://packagist.org', - exposes_secrets: true + exposes_secrets: true, + required: false validates :username, presence: true, if: :activated? validates :token, presence: true, if: :activated? @@ -55,8 +56,8 @@ module Integrations begin result = execute(data) return { success: false, result: result[:message] } if result[:http_status] != 202 - rescue StandardError => error - return { success: false, result: error } + rescue StandardError => e + return { success: false, result: e } end { success: true, result: result[:message] } diff --git a/app/models/integrations/pipelines_email.rb b/app/models/integrations/pipelines_email.rb index 77cbba25f2c..55a8ce0be11 100644 --- a/app/models/integrations/pipelines_email.rb +++ b/app/models/integrations/pipelines_email.rb @@ -84,8 +84,8 @@ module Integrations result = execute(data, force: true) { success: true, result: result } - rescue StandardError => error - { success: false, result: error } + rescue StandardError => e + { success: false, result: e } end def should_pipeline_be_notified?(data) diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index e672a985810..142f466018b 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -70,8 +70,8 @@ module Integrations prometheus_client.ping { success: true, result: 'Checked API endpoint' } - rescue Gitlab::PrometheusClient::Error => err - { success: false, result: err } + rescue Gitlab::PrometheusClient::Error => e + { success: false, result: e } end def prometheus_client diff --git a/app/models/integrations/pumble.rb b/app/models/integrations/pumble.rb new file mode 100644 index 00000000000..17026410eb1 --- /dev/null +++ b/app/models/integrations/pumble.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Integrations + class Pumble < BaseChatNotification + def title + 'Pumble' + end + + def description + s_("PumbleIntegration|Send notifications about project events to Pumble.") + end + + def self.to_param + 'pumble' + end + + def help + docs_link = ActionController::Base.helpers.link_to( + _('Learn more.'), + Rails.application.routes.url_helpers.help_page_url('user/project/integrations/pumble'), + target: '_blank', + rel: 'noopener noreferrer' + ) + # rubocop:disable Layout/LineLength + s_("PumbleIntegration|Send notifications about project events to Pumble. %{docs_link}") % { docs_link: docs_link.html_safe } + # rubocop:enable Layout/LineLength + end + + def default_channel_placeholder + end + + def self.supported_events + %w[push issue confidential_issue merge_request note confidential_note tag_push + pipeline wiki_page] + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "https://api.pumble.com/workspaces/x/...", required: true }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: self.class.branch_choices + } + ] + end + + private + + def notify(message, opts) + header = { 'Content-Type' => 'application/json' } + response = Gitlab::HTTP.post(webhook, headers: header, body: { text: message.summary }.to_json) + + response if response.success? + end + end +end diff --git a/app/models/integrations/slack.rb b/app/models/integrations/slack.rb index 93263229109..c254ea379bb 100644 --- a/app/models/integrations/slack.rb +++ b/app/models/integrations/slack.rb @@ -9,6 +9,7 @@ module Integrations push issue confidential_issue merge_request note confidential_note tag_push wiki_page deployment ].freeze + SNOWPLOW_EVENT_CATEGORY = self.name prop_accessor EVENT_CHANNEL['alert'] @@ -54,6 +55,22 @@ module Integrations key = "i_ecosystem_slack_service_#{event}_notification" Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user_id) + + return unless Feature.enabled?(:route_hll_to_snowplow_phase2) + + optional_arguments = { + project: project, + namespace: group || project&.namespace + }.compact + + Gitlab::Tracking.event( + SNOWPLOW_EVENT_CATEGORY, + Integration::SNOWPLOW_EVENT_ACTION, + label: Integration::SNOWPLOW_EVENT_LABEL, + property: key, + user: User.find(user_id), + **optional_arguments + ) end override :configurable_channels? diff --git a/app/models/integrations/teamcity.rb b/app/models/integrations/teamcity.rb index e0299c9ac5f..ca7a715f4b3 100644 --- a/app/models/integrations/teamcity.rb +++ b/app/models/integrations/teamcity.rb @@ -67,11 +67,11 @@ module Integrations end def build_page(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:build_page] } + with_reactive_cache(sha, ref) { |cached| cached[:build_page] } end def commit_status(sha, ref) - with_reactive_cache(sha, ref) {|cached| cached[:commit_status] } + with_reactive_cache(sha, ref) { |cached| cached[:commit_status] } end def calculate_reactive_cache(sha, ref) diff --git a/app/models/issuable_severity.rb b/app/models/issuable_severity.rb index 928301e1da6..cd7e5fafb60 100644 --- a/app/models/issuable_severity.rb +++ b/app/models/issuable_severity.rb @@ -3,18 +3,18 @@ class IssuableSeverity < ApplicationRecord DEFAULT = 'unknown' SEVERITY_LABELS = { - unknown: 'Unknown', - low: 'Low - S4', - medium: 'Medium - S3', - high: 'High - S2', + unknown: 'Unknown', + low: 'Low - S4', + medium: 'Medium - S3', + high: 'High - S2', critical: 'Critical - S1' }.freeze SEVERITY_QUICK_ACTION_PARAMS = { - unknown: %w(Unknown 0), - low: %w(Low S4 4), - medium: %w(Medium S3 3), - high: %w(High S2 2), + unknown: %w(Unknown 0), + low: %w(Low S4 4), + medium: %w(Medium S3 3), + high: %w(High S2 2), critical: %w(Critical S1 1) }.freeze diff --git a/app/models/issue.rb b/app/models/issue.rb index cae42115bef..4114467eb25 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -99,6 +99,10 @@ class Issue < ApplicationRecord validates :project, presence: true validates :issue_type, presence: true validates :namespace, presence: true, if: -> { project.present? } + validates :work_item_type, presence: true + + validate :due_date_after_start_date + validate :parent_link_confidentiality enum issue_type: WorkItems::Type.base_types @@ -201,7 +205,7 @@ class Issue < ApplicationRecord scope :with_null_relative_position, -> { where(relative_position: nil) } scope :with_non_null_relative_position, -> { where.not(relative_position: nil) } - before_validation :ensure_namespace_id + before_validation :ensure_namespace_id, :ensure_work_item_type after_commit :expire_etag_cache, unless: :importing? after_save :ensure_metrics, unless: :importing? @@ -257,17 +261,17 @@ class Issue < ApplicationRecord order = ::Gitlab::Pagination::Keyset::Order.build([ ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( attribute_name: attribute_name, - column_expression: column, - order_expression: column.send(direction).send(nullable), - reversed_order_expression: column.send(reversed_direction).send(nullable), - order_direction: direction, - distinct: false, - add_to_projections: true, - nullable: nullable + column_expression: column, + order_expression: column.send(direction).send(nullable), + reversed_order_expression: column.send(reversed_direction).send(nullable), + order_direction: direction, + distinct: false, + add_to_projections: true, + nullable: nullable ), ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( attribute_name: 'id', - order_expression: arel_table['id'].desc + order_expression: arel_table['id'].desc ) ]) # rubocop: enable GitlabSecurity/PublicSend @@ -289,6 +293,16 @@ class Issue < ApplicationRecord def pg_full_text_search(search_term) super.where('issue_search_data.project_id = issues.project_id') end + + override :full_search + def full_search(query, matched_columns: nil, use_minimum_char_limit: true) + return super if query.match?(IssuableFinder::FULL_TEXT_SEARCH_TERM_REGEX) + + super.where( + 'issues.title NOT SIMILAR TO :pattern OR issues.description NOT SIMILAR TO :pattern', + pattern: IssuableFinder::FULL_TEXT_SEARCH_TERM_PATTERN + ) + end end def next_object_by_relative_position(ignoring: nil, order: :asc) @@ -660,6 +674,29 @@ class Issue < ApplicationRecord private + def due_date_after_start_date + return unless start_date.present? && due_date.present? + + if due_date < start_date + errors.add(:due_date, 'must be greater than or equal to start date') + end + end + + # Although parent/child relationship can be set only for WorkItems, we + # still need to validate it for Issue model too, because both models use + # same table. + def parent_link_confidentiality + return unless persisted? + + if confidential? && WorkItems::ParentLink.has_public_children?(id) + errors.add(:confidential, _('confidential parent can not be used if there are non-confidential children.')) + end + + if !confidential? && WorkItems::ParentLink.has_confidential_parent?(id) + errors.add(:confidential, _('associated parent is confidential and can not have non-confidential children.')) + end + end + override :persist_pg_full_text_search_vector def persist_pg_full_text_search_vector(search_vector) Issues::SearchData.upsert({ project_id: project_id, issue_id: id, search_vector: search_vector }, unique_by: %i(project_id issue_id)) @@ -696,6 +733,12 @@ class Issue < ApplicationRecord def ensure_namespace_id self.namespace = project.project_namespace if project end + + def ensure_work_item_type + return if work_item_type_id.present? || work_item_type_id_change&.last.present? + + self.work_item_type = WorkItems::Type.default_by_type(issue_type) + end end Issue.prepend_mod_with('Issue') diff --git a/app/models/jira_connect_installation.rb b/app/models/jira_connect_installation.rb index e34543534f3..8befe9a9230 100644 --- a/app/models/jira_connect_installation.rb +++ b/app/models/jira_connect_installation.rb @@ -2,9 +2,9 @@ class JiraConnectInstallation < ApplicationRecord attr_encrypted :shared_secret, - mode: :per_attribute_iv, + mode: :per_attribute_iv, algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32 + key: Settings.attr_encrypted_db_key_base_32 has_many :subscriptions, class_name: 'JiraConnectSubscription' diff --git a/app/models/key.rb b/app/models/key.rb index 9f6029cc5d4..78b0a38bcaa 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -40,6 +40,7 @@ class Key < ApplicationRecord after_destroy :refresh_user_cache alias_attribute :fingerprint_md5, :fingerprint + alias_attribute :name, :title scope :preload_users, -> { preload(:user) } scope :for_user, -> (user) { where(user: user) } diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index caeffae7bda..8aa48561e60 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -15,7 +15,7 @@ class LfsObject < ApplicationRecord scope :for_oids, -> (oids) { where(oid: oids) } scope :for_oid_and_size, -> (oid, size) { find_by(oid: oid, size: size) } - validates :oid, presence: true, uniqueness: true + validates :oid, presence: true, uniqueness: true, format: { with: /\A\h{64}\z/ } mount_file_store_uploader LfsObjectUploader diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index 6dfd6ea2aae..94444f4b6d3 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -9,26 +9,23 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel self.ignored_columns = %i[partition] partitioned_by :partition, strategy: :sliding_list, - next_partition_if: -> (active_partition) do - return false if Feature.disabled?(:lfk_automatic_partition_creation) - - oldest_record_in_partition = LooseForeignKeys::DeletedRecord - .select(:id, :created_at) - .for_partition(active_partition) - .order(:id) - .limit(1) - .take - - oldest_record_in_partition.present? && oldest_record_in_partition.created_at < PARTITION_DURATION.ago - end, - detach_partition_if: -> (partition) do - return false if Feature.disabled?(:lfk_automatic_partition_dropping) - - !LooseForeignKeys::DeletedRecord - .for_partition(partition) - .status_pending - .exists? - end + next_partition_if: -> (active_partition) do + oldest_record_in_partition = LooseForeignKeys::DeletedRecord + .select(:id, :created_at) + .for_partition(active_partition) + .order(:id) + .limit(1) + .take + + oldest_record_in_partition.present? && + oldest_record_in_partition.created_at < PARTITION_DURATION.ago + end, + detach_partition_if: -> (partition) do + !LooseForeignKeys::DeletedRecord + .for_partition(partition) + .status_pending + .exists? + end scope :for_table, -> (table) { where(fully_qualified_table_name: table) } scope :for_partition, -> (partition) { where(partition: partition) } diff --git a/app/models/member.rb b/app/models/member.rb index dcca63b5691..0cd1e022617 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -28,6 +28,7 @@ class Member < ApplicationRecord belongs_to :user belongs_to :source, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :member_namespace, inverse_of: :namespace_members, foreign_key: 'member_namespace_id', class_name: 'Namespace' + belongs_to :member_role has_one :member_task delegate :name, :username, :email, :last_activity_on, to: :user, prefix: true @@ -58,6 +59,7 @@ class Member < ApplicationRecord }, if: :project_bot? validate :access_level_inclusion + validate :validate_member_role_access_level scope :with_invited_user_state, -> do joins('LEFT JOIN users as invited_user ON invited_user.email = members.invite_email') @@ -428,6 +430,14 @@ class Member < ApplicationRecord errors.add(:access_level, "is not included in the list") end + def validate_member_role_access_level + return unless member_role_id + + if access_level != member_role.base_access_level + errors.add(:member_role_id, _("role's base access level does not match the access level of the membership")) + end + end + def send_invite # override in subclass end @@ -455,6 +465,8 @@ class Member < ApplicationRecord # transaction has been committed, resulting in the job either throwing an # error or not doing any meaningful work. # rubocop: disable CodeReuse/ServiceClass + + # This method is overridden in the test environment, see stubbed_member.rb def refresh_member_authorized_projects(blocking:) UserProjectAccessChangedService.new(user_id).execute(blocking: blocking) end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 87af6a9a7f7..2b35f7da7b4 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -7,7 +7,6 @@ class GroupMember < Member SOURCE_TYPE = 'Namespace' SOURCE_TYPE_FORMAT = /\ANamespace\z/.freeze - THRESHOLD_FOR_REFRESHING_AUTHORIZATIONS_VIA_PROJECTS = 1000 belongs_to :group, foreign_key: 'source_id' alias_attribute :namespace_id, :source_id @@ -67,28 +66,8 @@ class GroupMember < Member # its projects are also destroyed, so the removal of project_authorizations # will happen behind the scenes via DB foreign keys anyway. return if destroyed_by_association.present? - return unless user_id - return super if Feature.disabled?(:refresh_authorizations_via_affected_projects_on_group_membership, group) - # rubocop:disable CodeReuse/ServiceClass - projects_to_refresh = Groups::ProjectsRequiringAuthorizationsRefresh::OnDirectMembershipFinder.new(group).execute - threshold_exceeded = (projects_to_refresh.size > THRESHOLD_FOR_REFRESHING_AUTHORIZATIONS_VIA_PROJECTS) - - # We want to try the new approach only if the number of affected projects are greater than the set threshold. - return super unless threshold_exceeded - - AuthorizedProjectUpdate::ProjectAccessChangedService - .new(projects_to_refresh) - .execute(blocking: false) - - # Until we compare the inconsistency rates of the new approach - # the old approach, we still run AuthorizedProjectsWorker - # but with some delay and lower urgency as a safety net. - UserProjectAccessChangedService - .new(user_id) - .execute(blocking: false, priority: UserProjectAccessChangedService::LOW_PRIORITY) - - # rubocop:enable CodeReuse/ServiceClass + super end def send_invite diff --git a/app/models/members/last_group_owner_assigner.rb b/app/models/members/last_group_owner_assigner.rb index c85116858c7..e411a0ef5eb 100644 --- a/app/models/members/last_group_owner_assigner.rb +++ b/app/models/members/last_group_owner_assigner.rb @@ -8,7 +8,7 @@ class LastGroupOwnerAssigner end def execute - @last_blocked_owner = no_owners_in_heirarchy? && group.single_blocked_owner? + @last_blocked_owner = no_owners_in_hierarchy? && group.single_blocked_owner? @group_single_owner = owners.size == 1 members.each { |member| set_last_owner(member) } @@ -18,7 +18,7 @@ class LastGroupOwnerAssigner attr_reader :group, :members, :last_blocked_owner, :group_single_owner - def no_owners_in_heirarchy? + def no_owners_in_hierarchy? owners.empty? end diff --git a/app/models/members/member_role.rb b/app/models/members/member_role.rb new file mode 100644 index 00000000000..2e8532fa739 --- /dev/null +++ b/app/models/members/member_role.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass + has_many :members + belongs_to :namespace + + validates :namespace_id, presence: true + validates :base_access_level, presence: true +end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index c97f00364fd..8fd82fcb34a 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -111,7 +111,7 @@ class ProjectMember < Member # rubocop:disable CodeReuse/ServiceClass if blocking - AuthorizedProjectUpdate::ProjectRecalculatePerUserWorker.bulk_perform_and_wait([[project.id, user.id]]) + blocking_project_authorizations_refresh else AuthorizedProjectUpdate::ProjectRecalculatePerUserWorker.perform_async(project.id, user.id) end @@ -124,6 +124,11 @@ class ProjectMember < Member # rubocop:enable CodeReuse/ServiceClass end + # This method is overridden in the test environment, see stubbed_member.rb + def blocking_project_authorizations_refresh + AuthorizedProjectUpdate::ProjectRecalculatePerUserWorker.bulk_perform_and_wait([[project.id, user.id]]) + end + # TODO: https://gitlab.com/groups/gitlab-org/-/epics/7054 # temporary until we can we properly remove the source columns override :set_member_namespace_id diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index ec97ab0ea42..3c06e1aa983 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -37,23 +37,25 @@ class MergeRequest < ApplicationRecord SORTING_PREFERENCE_FIELD = :merge_requests_sort ALLOWED_TO_USE_MERGE_BASE_PIPELINE_FOR_COMPARISON = { - 'Ci::CompareMetricsReportsService' => ->(project) { true }, + 'Ci::CompareMetricsReportsService' => ->(project) { true }, 'Ci::CompareCodequalityReportsService' => ->(project) { true } }.freeze + MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS = 100 + belongs_to :target_project, class_name: "Project" belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" belongs_to :iteration, foreign_key: 'sprint_id' has_internal_id :iid, scope: :target_project, track_if: -> { !importing? }, - init: ->(mr, scope) do - if mr - mr.target_project&.merge_requests&.maximum(:iid) - elsif scope[:project] - where(target_project: scope[:project]).maximum(:iid) - end - end + init: ->(mr, scope) do + if mr + mr.target_project&.merge_requests&.maximum(:iid) + elsif scope[:project] + where(target_project: scope[:project]).maximum(:iid) + end + end has_many :merge_request_diffs, -> { regular }, inverse_of: :merge_request @@ -121,7 +123,8 @@ class MergeRequest < ApplicationRecord :force_remove_source_branch, :commit_message, :squash_commit_message, - :sha + :sha, + :skip_ci ].freeze serialize :merge_params, Hash # rubocop:disable Cop/ActiveRecordSerialize @@ -263,6 +266,7 @@ class MergeRequest < ApplicationRecord validate :validate_branches, unless: [:allow_broken, :importing?, :closed_or_merged_without_fork?] validate :validate_fork, unless: :closed_or_merged_without_fork? validate :validate_target_project, on: :create + validate :validate_reviewer_and_assignee_size_length, unless: :importing? scope :by_source_or_target_branch, ->(branch_name) do where("source_branch = :branch OR target_branch = :branch", branch: branch_name) @@ -427,8 +431,7 @@ class MergeRequest < ApplicationRecord def self.total_time_to_merge join_metrics .merge(MergeRequest::Metrics.with_valid_time_to_merge) - .pluck(MergeRequest::Metrics.time_to_merge_expression) - .first + .pick(MergeRequest::Metrics.time_to_merge_expression) end after_save :keep_around_commit, unless: :importing? @@ -927,9 +930,9 @@ class MergeRequest < ApplicationRecord # most recent data possible. def repository_diff_refs Gitlab::Diff::DiffRefs.new( - base_sha: branch_merge_base_sha, + base_sha: branch_merge_base_sha, start_sha: target_branch_sha, - head_sha: source_branch_sha + head_sha: source_branch_sha ) end @@ -992,6 +995,20 @@ class MergeRequest < ApplicationRecord 'Source project is not a fork of the target project' end + def self.max_number_of_assignees_or_reviewers_message + # Assignees will be included in https://gitlab.com/gitlab-org/gitlab/-/issues/368936 + _("total must be less than or equal to %{size}") % { size: MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS } + end + + def validate_reviewer_and_assignee_size_length + # Assigness will be added in a subsequent MR https://gitlab.com/gitlab-org/gitlab/-/issues/368936 + return true unless Feature.enabled?(:limit_reviewer_and_assignee_size) + return true unless reviewers.size > MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS + + errors.add :reviewers, + -> (_object, _data) { MergeRequest.max_number_of_assignees_or_reviewers_message } + end + def merge_ongoing? # While the MergeRequest is locked, it should present itself as 'merge ongoing'. # The unlocking process is handled by StuckMergeJobsWorker scheduled in Cron. @@ -1170,17 +1187,30 @@ class MergeRequest < ApplicationRecord ] end + def detailed_merge_status + if cannot_be_merged_rechecking? || preparing? || checking? + return :checking + elsif unchecked? + return :unchecked + end + + checks = execute_merge_checks + + if checks.success? + :mergeable + else + checks.failure_reason + end + end + # rubocop: disable CodeReuse/ServiceClass def mergeable_state?(skip_ci_check: false, skip_discussions_check: false) if Feature.enabled?(:improved_mergeability_checks, self.project) - additional_checks = MergeRequests::Mergeability::RunChecksService.new( - merge_request: self, - params: { - skip_ci_check: skip_ci_check, - skip_discussions_check: skip_discussions_check - } - ) - additional_checks.execute.all?(&:success?) + additional_checks = execute_merge_checks(params: { + skip_ci_check: skip_ci_check, + skip_discussions_check: skip_discussions_check + }) + additional_checks.execute.success? else return false unless open? return false if draft? @@ -1500,14 +1530,14 @@ class MergeRequest < ApplicationRecord end def self.merge_train_ref?(ref) - %r{\Arefs/#{Repository::REF_MERGE_REQUEST}/\d+/train\z}.match?(ref) + %r{\Arefs/#{Repository::REF_MERGE_REQUEST}/\d+/train\z}o.match?(ref) end def in_locked_state lock_mr yield ensure - unlock_mr + unlock_mr if locked? end def update_and_mark_in_progress_merge_commit_sha(commit_id) @@ -1985,6 +2015,10 @@ class MergeRequest < ApplicationRecord target_branch == project.default_branch end + def merge_blocked_by_other_mrs? + false # Overridden in EE + end + private attr_accessor :skip_fetch_ref @@ -2038,6 +2072,12 @@ class MergeRequest < ApplicationRecord def report_type_enabled?(report_type) !!actual_head_pipeline&.batch_lookup_report_artifact_for_file_type(report_type) end + + def execute_merge_checks(params: {}) + # rubocop: disable CodeReuse/ServiceClass + MergeRequests::Mergeability::RunChecksService.new(merge_request: self, params: params).execute + # rubocop: enable CodeReuse/ServiceClass + end end MergeRequest.prepend_mod_with('MergeRequest') diff --git a/app/models/merge_request/approval_removal_settings.rb b/app/models/merge_request/approval_removal_settings.rb new file mode 100644 index 00000000000..b07242e2578 --- /dev/null +++ b/app/models/merge_request/approval_removal_settings.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class MergeRequest::ApprovalRemovalSettings # rubocop:disable Style/ClassAndModuleChildren + include ActiveModel::Validations + + attr_accessor :project + + validate :mutually_exclusive_settings + + def initialize(project, reset_approvals_on_push, selective_code_owner_removals) + @project = project + @reset_approvals_on_push = reset_approvals_on_push + @selective_code_owner_removals = selective_code_owner_removals + end + + private + + def selective_code_owner_removals + if @selective_code_owner_removals.nil? + project.project_setting.selective_code_owner_removals + else + @selective_code_owner_removals + end + end + + def reset_approvals_on_push + if @reset_approvals_on_push.nil? + project.reset_approvals_on_push + else + @reset_approvals_on_push + end + end + + def mutually_exclusive_settings + return unless selective_code_owner_removals && reset_approvals_on_push + + errors.add(:base, 'selective_code_owner_removals can only be enabled when reset_approvals_on_push is disabled') + end +end diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb index b984228eb13..c546a5a0025 100644 --- a/app/models/merge_request/metrics.rb +++ b/app/models/merge_request/metrics.rb @@ -41,8 +41,7 @@ class MergeRequest::Metrics < ApplicationRecord def self.total_time_to_merge with_valid_time_to_merge - .pluck(time_to_merge_expression) - .first + .pick(time_to_merge_expression) end end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index e08b2cc2a7d..9f7e98dc04b 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -358,9 +358,9 @@ class MergeRequestDiff < ApplicationRecord return unless start_commit_sha || base_commit_sha Gitlab::Diff::DiffRefs.new( - base_sha: base_commit_sha, + base_sha: base_commit_sha, start_sha: start_commit_sha, - head_sha: head_commit_sha + head_sha: head_commit_sha ) end @@ -381,9 +381,9 @@ class MergeRequestDiff < ApplicationRecord likely_base_commit_sha = (first_commit&.parent || first_commit)&.sha Gitlab::Diff::DiffRefs.new( - base_sha: likely_base_commit_sha, + base_sha: likely_base_commit_sha, start_sha: safe_start_commit_sha, - head_sha: head_commit_sha + head_sha: head_commit_sha ) end @@ -706,8 +706,7 @@ class MergeRequestDiff < ApplicationRecord latest_id = MergeRequest .where(id: merge_request_id) .limit(1) - .pluck(:latest_merge_request_diff_id) - .first + .pick(:latest_merge_request_diff_id) latest_id && self.id < latest_id end diff --git a/app/models/ml.rb b/app/models/ml.rb new file mode 100644 index 00000000000..e426ce851eb --- /dev/null +++ b/app/models/ml.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module Ml + def self.table_name_prefix + 'ml_' + end +end diff --git a/app/models/ml/candidate.rb b/app/models/ml/candidate.rb new file mode 100644 index 00000000000..e181217f01c --- /dev/null +++ b/app/models/ml/candidate.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Ml + class Candidate < ApplicationRecord + validates :iid, :experiment, presence: true + + belongs_to :experiment, class_name: 'Ml::Experiment' + belongs_to :user + has_many :metrics, class_name: 'Ml::CandidateMetric' + has_many :params, class_name: 'Ml::CandidateParam' + end +end diff --git a/app/models/ml/candidate_metric.rb b/app/models/ml/candidate_metric.rb new file mode 100644 index 00000000000..e03a8b83ee6 --- /dev/null +++ b/app/models/ml/candidate_metric.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Ml + class CandidateMetric < ApplicationRecord + validates :candidate, presence: true + validates :name, length: { maximum: 250 }, presence: true + + belongs_to :candidate, class_name: 'Ml::Candidate' + end +end diff --git a/app/models/ml/candidate_param.rb b/app/models/ml/candidate_param.rb new file mode 100644 index 00000000000..cbdddcc8a1a --- /dev/null +++ b/app/models/ml/candidate_param.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Ml + class CandidateParam < ApplicationRecord + validates :candidate, presence: true + validates :name, :value, length: { maximum: 250 }, presence: true + + belongs_to :candidate, class_name: 'Ml::Candidate' + end +end diff --git a/app/models/ml/experiment.rb b/app/models/ml/experiment.rb new file mode 100644 index 00000000000..7ef9c70ba7e --- /dev/null +++ b/app/models/ml/experiment.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Ml + class Experiment < ApplicationRecord + validates :name, :iid, :project, presence: true + validates :iid, :name, uniqueness: { scope: :project, message: "should be unique in the project" } + + belongs_to :project + belongs_to :user + has_many :candidates, class_name: 'Ml::Candidate' + end +end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index f23a859b119..06f49f16d66 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -40,15 +40,21 @@ class Namespace < ApplicationRecord PATH_TRAILING_VIOLATIONS = %w[.git .atom .].freeze + # The first date in https://docs.gitlab.com/ee/user/usage_quotas.html#namespace-storage-limit-enforcement-schedule + # Determines when we start enforcing namespace storage + MIN_STORAGE_ENFORCEMENT_DATE = Date.new(2022, 10, 19) + cache_markdown_field :description, pipeline: :description has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics has_one :namespace_settings, inverse_of: :namespace, class_name: 'NamespaceSetting', autosave: true has_one :ci_cd_settings, inverse_of: :namespace, class_name: 'NamespaceCiCdSetting', autosave: true + has_one :namespace_details, inverse_of: :namespace, class_name: 'Namespace::Detail', autosave: true has_one :namespace_statistics has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' + has_many :member_roles has_many :runner_namespaces, inverse_of: :namespace, class_name: 'Ci::RunnerNamespace' has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' @@ -77,6 +83,8 @@ class Namespace < ApplicationRecord has_many :work_items, inverse_of: :namespace has_many :issues, inverse_of: :namespace + has_many :timelog_categories, class_name: 'TimeTracking::TimelogCategory' + validates :owner, presence: true, if: ->(n) { n.owner_required? } validates :name, presence: true, @@ -120,6 +128,7 @@ class Namespace < ApplicationRecord to: :namespace_settings, allow_nil: true after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_parent_id? } + after_save :reload_namespace_details after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } @@ -559,9 +568,7 @@ class Namespace < ApplicationRecord def storage_enforcement_date return Date.current if Feature.enabled?(:namespace_storage_limit_bypass_date_check, self) - # should return something like Date.new(2022, 02, 03) - # TBD: https://gitlab.com/gitlab-org/gitlab/-/issues/350632 - nil + MIN_STORAGE_ENFORCEMENT_DATE end def certificate_based_clusters_enabled? @@ -671,6 +678,12 @@ class Namespace < ApplicationRecord end end + def reload_namespace_details + return unless !project_namespace? && (previous_changes.keys & %w(description description_html cached_markdown_version)).any? && namespace_details.present? + + namespace_details.reset + end + def sync_share_with_group_lock_with_parent if parent&.share_with_group_lock? self.share_with_group_lock = true diff --git a/app/models/namespace/detail.rb b/app/models/namespace/detail.rb new file mode 100644 index 00000000000..dbbf9f4944a --- /dev/null +++ b/app/models/namespace/detail.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class Namespace::Detail < ApplicationRecord + belongs_to :namespace, inverse_of: :namespace_details + validates :namespace, presence: true + validates :description, length: { maximum: 255 } + + self.primary_key = :namespace_id +end diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 6f404ec12d0..81ac026d7ff 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -27,15 +27,9 @@ module Namespaces def self_and_ancestors(include_self: true, upto: nil, hierarchy_order: nil) return super unless use_traversal_ids_for_ancestor_scopes? - if Feature.enabled?(:use_traversal_ids_for_ancestor_scopes_with_inner_join) - self_and_ancestors_from_inner_join(include_self: include_self, - upto: upto, hierarchy_order: - hierarchy_order) - else - self_and_ancestors_from_ancestors_cte(include_self: include_self, - upto: upto, - hierarchy_order: hierarchy_order) - end + self_and_ancestors_from_inner_join(include_self: include_self, + upto: upto, hierarchy_order: + hierarchy_order) end def self_and_ancestor_ids(include_self: true) @@ -117,37 +111,6 @@ module Namespaces use_traversal_ids? end - def self_and_ancestors_from_ancestors_cte(include_self: true, upto: nil, hierarchy_order: nil) - base_cte = all.select('namespaces.id', 'namespaces.traversal_ids').as_cte(:base_ancestors_cte) - - # We have to alias id with 'AS' to avoid ambiguous column references by calling methods. - ancestors_cte = unscoped - .unscope(where: [:type]) - .select('id as base_id', - "#{unnest_func(base_cte.table['traversal_ids']).to_sql} as ancestor_id") - .from(base_cte.table) - .as_cte(:ancestors_cte) - - namespaces = Arel::Table.new(:namespaces) - - records = unscoped - .with(base_cte.to_arel, ancestors_cte.to_arel) - .distinct - .from([ancestors_cte.table, namespaces]) - .where(namespaces[:id].eq(ancestors_cte.table[:ancestor_id])) - .order_by_depth(hierarchy_order) - - unless include_self - records = records.where(ancestors_cte.table[:base_id].not_eq(ancestors_cte.table[:ancestor_id])) - end - - if upto - records = records.where.not(id: unscoped.where(id: upto).select('unnest(traversal_ids)')) - end - - records - end - def self_and_ancestors_from_inner_join(include_self: true, upto: nil, hierarchy_order: nil) base_cte = all.reselect('namespaces.traversal_ids').as_cte(:base_ancestors_cte) @@ -181,25 +144,15 @@ module Namespaces end def self_and_descendants_with_comparison_operators(include_self: true) - base = all.select(:traversal_ids) - base = base.select(:id) if Feature.enabled?(:linear_scopes_superset) + base = all.select(:id, :traversal_ids) base_cte = base.as_cte(:descendants_base_cte) namespaces = Arel::Table.new(:namespaces) - withs = [base_cte.to_arel] - froms = [] - - if Feature.enabled?(:linear_scopes_superset) - superset_cte = self.superset_cte(base_cte.table.name) - withs += [superset_cte.to_arel] - froms = [superset_cte.table] - else - froms = [base_cte.table] - end - + superset_cte = self.superset_cte(base_cte.table.name) + withs = [base_cte.to_arel, superset_cte.to_arel] # Order is important. namespace should be last to handle future joins. - froms += [namespaces] + froms = [superset_cte.table, namespaces] base_ref = froms.first diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb index 560ff861105..a034d97a6bb 100644 --- a/app/models/network/graph.rb +++ b/app/models/network/graph.rb @@ -211,7 +211,7 @@ module Network # Visit branching chains leaves.each do |l| - parents = l.parents(@map).select {|p| p.space == 0} + parents = l.parents(@map).select { |p| p.space == 0 } parents.each do |p| place_chain(p, l.time) end diff --git a/app/models/note.rb b/app/models/note.rb index 986a85acac6..1715f7cdc3b 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -23,6 +23,8 @@ class Note < ApplicationRecord include FromUnion include Sortable + ISSUE_TASK_SYSTEM_NOTE_PATTERN = /\A.*marked\sthe\stask.+as\s(completed|incomplete).*\z/.freeze + cache_markdown_field :note, pipeline: :note, issuable_reference_expansion_enabled: true redact_field :note @@ -685,6 +687,22 @@ class Note < ApplicationRecord Ability.users_that_can_read_internal_notes(users, resource_parent).pluck(:id) end + # Method necesary while we transition into the new format for task system notes + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/369923 + def note + return super unless system? && for_issue? && super.match?(ISSUE_TASK_SYSTEM_NOTE_PATTERN) + + super.sub!('task', 'checklist item') + end + + # Method necesary while we transition into the new format for task system notes + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/369923 + def note_html + return super unless system? && for_issue? && super.match?(ISSUE_TASK_SYSTEM_NOTE_PATTERN) + + super.sub!('task', 'checklist item') + end + private def system_note_viewable_by?(user) diff --git a/app/models/notification_reason.rb b/app/models/notification_reason.rb index 3713be6cb91..c227626af9e 100644 --- a/app/models/notification_reason.rb +++ b/app/models/notification_reason.rb @@ -6,7 +6,6 @@ class NotificationReason OWN_ACTIVITY = 'own_activity' ASSIGNED = 'assigned' REVIEW_REQUESTED = 'review_requested' - ATTENTION_REQUESTED = 'attention_requested' MENTIONED = 'mentioned' SUBSCRIBED = 'subscribed' @@ -15,7 +14,6 @@ class NotificationReason OWN_ACTIVITY, ASSIGNED, REVIEW_REQUESTED, - ATTENTION_REQUESTED, MENTIONED, SUBSCRIBED ].freeze diff --git a/app/models/oauth_access_token.rb b/app/models/oauth_access_token.rb index 20130f01d44..7d71e15d3c5 100644 --- a/app/models/oauth_access_token.rb +++ b/app/models/oauth_access_token.rb @@ -6,7 +6,6 @@ class OauthAccessToken < Doorkeeper::AccessToken alias_attribute :user, :resource_owner - scope :distinct_resource_owner_counts, ->(applications) { where(application: applications).distinct.group(:application_id).count(:resource_owner_id) } scope :latest_per_application, -> { select('distinct on(application_id) *').order(application_id: :desc, created_at: :desc) } scope :preload_application, -> { preload(:application) } @@ -17,4 +16,14 @@ class OauthAccessToken < Doorkeeper::AccessToken super end end + + # this method overrides a shortcoming upstream, more context: + # https://gitlab.com/gitlab-org/gitlab/-/issues/367888 + def self.find_by_fallback_token(attr, plain_secret) + return unless fallback_secret_strategy && fallback_secret_strategy == Doorkeeper::SecretStoring::Plain + # token is hashed, don't allow plaintext comparison + return if plain_secret.starts_with?("$") + + super + end end diff --git a/app/models/operations/feature_flag.rb b/app/models/operations/feature_flag.rb index 7db396bcad5..e36c59366fe 100644 --- a/app/models/operations/feature_flag.rb +++ b/app/models/operations/feature_flag.rb @@ -42,7 +42,7 @@ module Operations scope :enabled, -> { where(active: true) } scope :disabled, -> { where(active: false) } - scope :new_version_only, -> { where(version: :new_version_flag)} + scope :new_version_only, -> { where(version: :new_version_flag) } enum version: { new_version_flag: 2 diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 90a1bb4bc69..afd55b4f143 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -65,7 +65,7 @@ class Packages::Package < ApplicationRecord validates :name, uniqueness: { scope: %i[project_id version package_type], - conditions: -> { not_pending_destruction} + conditions: -> { not_pending_destruction } }, unless: -> { pending_destruction? || conan? || debian_package? } @@ -327,7 +327,7 @@ class Packages::Package < ApplicationRecord def normalized_pypi_name return name unless pypi? - name.gsub(/#{Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING}/, '-').downcase + name.gsub(/#{Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING}/o, '-').downcase end private diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 9e93bff4acf..2e25839c47f 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -23,10 +23,10 @@ class PagesDomain < ApplicationRecord validates :domain, uniqueness: { case_sensitive: false } validates :certificate, :key, presence: true, if: :usage_serverless? validates :certificate, presence: { message: 'must be present if HTTPS-only is enabled' }, - if: :certificate_should_be_present? + if: :certificate_should_be_present? validates :certificate, certificate: true, if: ->(domain) { domain.certificate.present? } validates :key, presence: { message: 'must be present if HTTPS-only is enabled' }, - if: :certificate_should_be_present? + if: :certificate_should_be_present? validates :key, certificate_key: true, named_ecdsa_key: true, if: ->(domain) { domain.key.present? } validates :verification_code, presence: true, allow_blank: false diff --git a/app/models/performance_monitoring/prometheus_dashboard.rb b/app/models/performance_monitoring/prometheus_dashboard.rb index 40d14aaa1de..4804f620a99 100644 --- a/app/models/performance_monitoring/prometheus_dashboard.rb +++ b/app/models/performance_monitoring/prometheus_dashboard.rb @@ -57,10 +57,10 @@ module PerformanceMonitoring self.class.from_json(reload_schema) [] - rescue Gitlab::Metrics::Dashboard::Errors::LayoutError => error - [error.message] - rescue ActiveModel::ValidationError => exception - exception.model.errors.map { |attr, error| "#{attr}: #{error}" } + rescue Gitlab::Metrics::Dashboard::Errors::LayoutError => e + [e.message] + rescue ActiveModel::ValidationError => e + e.model.errors.map { |attr, error| "#{attr}: #{error}" } end private diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 68ba3d6eab4..7e6e366f8da 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -20,7 +20,7 @@ class PersonalAccessToken < ApplicationRecord before_save :ensure_token - scope :active, -> { where("revoked = false AND (expires_at >= CURRENT_DATE OR expires_at IS NULL)") } + scope :active, -> { not_revoked.not_expired } scope :expiring_and_not_notified, ->(date) { where(["revoked = false AND expire_notification_delivered = false AND expires_at >= CURRENT_DATE AND expires_at <= ?", date]) } scope :expired_today_and_not_notified, -> { where(["revoked = false AND expires_at = CURRENT_DATE AND after_expiry_notification_delivered = false"]) } scope :inactive, -> { where("revoked = true OR expires_at < CURRENT_DATE") } @@ -33,6 +33,7 @@ class PersonalAccessToken < ApplicationRecord scope :preload_users, -> { preload(:user) } scope :order_expires_at_asc, -> { reorder(expires_at: :asc) } scope :order_expires_at_desc, -> { reorder(expires_at: :desc) } + scope :order_expires_at_asc_id_desc, -> { reorder(expires_at: :asc, id: :desc) } scope :project_access_token, -> { includes(:user).where(user: { user_type: :project_bot }) } scope :owner_is_human, -> { includes(:user).where(user: { user_type: :human }) } @@ -57,8 +58,8 @@ class PersonalAccessToken < ApplicationRecord begin Gitlab::CryptoHelper.aes256_gcm_decrypt(encrypted_token) - rescue StandardError => ex - logger.warn "Failed to decrypt #{self.name} value stored in Redis for key ##{redis_key}: #{ex.class}" + rescue StandardError => e + logger.warn "Failed to decrypt #{self.name} value stored in Redis for key ##{redis_key}: #{e.class}" encrypted_token end end @@ -77,7 +78,8 @@ class PersonalAccessToken < ApplicationRecord super.merge( { 'expires_at_asc' => -> { order_expires_at_asc }, - 'expires_at_desc' => -> { order_expires_at_desc } + 'expires_at_desc' => -> { order_expires_at_desc }, + 'expires_at_asc_id_desc' => -> { order_expires_at_asc_id_desc } } ) end diff --git a/app/models/preloaders/labels_preloader.rb b/app/models/preloaders/labels_preloader.rb index bb3206f5399..722d588d8bc 100644 --- a/app/models/preloaders/labels_preloader.rb +++ b/app/models/preloaders/labels_preloader.rb @@ -21,8 +21,8 @@ module Preloaders def preload_all preloader = ActiveRecord::Associations::Preloader.new - preloader.preload(labels.select {|l| l.is_a? ProjectLabel }, { project: [:project_feature, namespace: :route] }) - preloader.preload(labels.select {|l| l.is_a? GroupLabel }, { group: :route }) + preloader.preload(labels.select { |l| l.is_a? ProjectLabel }, { project: [:project_feature, namespace: :route] }) + preloader.preload(labels.select { |l| l.is_a? GroupLabel }, { group: :route }) labels.each do |label| label.lazy_subscription(user) label.lazy_subscription(user, project) if project.present? diff --git a/app/models/project.rb b/app/models/project.rb index ebfec34c3e1..0c49cc24a8d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -53,6 +53,7 @@ class Project < ApplicationRecord ignore_columns :mirror_last_update_at, :mirror_last_successful_update_at, remove_after: '2021-09-22', remove_with: '14.4' ignore_columns :pull_mirror_branch_prefix, remove_after: '2021-09-22', remove_with: '14.4' + ignore_columns :build_coverage_regex, remove_after: '2022-10-22', remove_with: '15.5' STATISTICS_ATTRIBUTE = 'repositories_count' UNKNOWN_IMPORT_URL = 'http://unknown.git' @@ -131,6 +132,8 @@ class Project < ApplicationRecord after_save :save_topics + after_save :reload_project_namespace_details + after_create -> { create_or_load_association(:project_feature) } after_create -> { create_or_load_association(:ci_cd_settings) } @@ -176,7 +179,7 @@ class Project < ApplicationRecord alias_method :parent, :namespace alias_attribute :parent_id, :namespace_id - has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event' + has_one :last_event, -> { order 'events.created_at DESC' }, class_name: 'Event' has_many :boards def self.integration_association_name(name) @@ -213,6 +216,7 @@ class Project < ApplicationRecord has_one :pipelines_email_integration, class_name: 'Integrations::PipelinesEmail' has_one :pivotaltracker_integration, class_name: 'Integrations::Pivotaltracker' has_one :prometheus_integration, class_name: 'Integrations::Prometheus', inverse_of: :project + has_one :pumble_integration, class_name: 'Integrations::Pumble' has_one :pushover_integration, class_name: 'Integrations::Pushover' has_one :redmine_integration, class_name: 'Integrations::Redmine' has_one :shimo_integration, class_name: 'Integrations::Shimo' @@ -288,6 +292,8 @@ class Project < ApplicationRecord has_many :project_members, -> { where(requested_at: nil) }, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_many :project_callouts, class_name: 'Users::ProjectCallout', foreign_key: :project_id + alias_method :members, :project_members has_many :users, through: :project_members @@ -446,7 +452,8 @@ class Project < ApplicationRecord :repository_access_level, :package_registry_access_level, :pages_access_level, :metrics_dashboard_access_level, :analytics_access_level, :operations_access_level, :security_and_compliance_access_level, - :container_registry_access_level, + :container_registry_access_level, :environments_access_level, :feature_flags_access_level, + :releases_access_level, to: :project_feature, allow_nil: true delegate :show_default_award_emojis, :show_default_award_emojis=, @@ -472,6 +479,7 @@ class Project < ApplicationRecord delegate :job_token_scope_enabled, :job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :keep_latest_artifact, :keep_latest_artifact=, to: :ci_cd_settings, allow_nil: true delegate :opt_in_jwt, :opt_in_jwt=, to: :ci_cd_settings, prefix: :ci, allow_nil: true + delegate :allow_fork_pipelines_to_run_in_parent_project, :allow_fork_pipelines_to_run_in_parent_project=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, to: :ci_cd_settings, allow_nil: true delegate :separated_caches, :separated_caches=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :runner_token_expiration_interval, :runner_token_expiration_interval=, :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval_human_readable=, to: :ci_cd_settings, allow_nil: true @@ -663,6 +671,7 @@ class Project < ApplicationRecord scope :imported_from, -> (type) { where(import_type: type) } scope :imported, -> { where.not(import_type: nil) } scope :with_enabled_error_tracking, -> { joins(:error_tracking_setting).where(project_error_tracking_settings: { enabled: true }) } + scope :last_activity_before, -> (time) { where('projects.last_activity_at < ?', time) } scope :with_service_desk_key, -> (key) do # project_key is not indexed for now @@ -814,7 +823,7 @@ class Project < ApplicationRecord (?<!#{Gitlab::PathRegex::PATH_START_CHAR}) ((?<namespace>#{Gitlab::PathRegex::FULL_NAMESPACE_FORMAT_REGEX})\/)? (?<project>#{Gitlab::PathRegex::PROJECT_PATH_FORMAT_REGEX}) - }x + }xo end def reference_postfix @@ -1041,6 +1050,7 @@ class Project < ApplicationRecord def emails_enabled? !emails_disabled? end + override :lfs_enabled? def lfs_enabled? return namespace.lfs_enabled? if self[:lfs_enabled].nil? @@ -1675,7 +1685,13 @@ class Project < ApplicationRecord end def has_active_hooks?(hooks_scope = :push_hooks) - hooks.hooks_for(hooks_scope).any? || SystemHook.hooks_for(hooks_scope).any? || Gitlab::FileHook.any? + @has_active_hooks ||= {} # rubocop: disable Gitlab/PredicateMemoization + + return @has_active_hooks[hooks_scope] if @has_active_hooks.key?(hooks_scope) + + @has_active_hooks[hooks_scope] = hooks.hooks_for(hooks_scope).any? || + SystemHook.hooks_for(hooks_scope).any? || + Gitlab::FileHook.any? end def has_active_integrations?(hooks_scope = :push_hooks) @@ -1757,8 +1773,8 @@ class Project < ApplicationRecord repository.after_create true - rescue StandardError => err - Gitlab::ErrorTracking.track_exception(err, project: { id: id, full_path: full_path, disk_path: disk_path }) + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, project: { id: id, full_path: full_path, disk_path: disk_path }) errors.add(:base, _('Failed to create repository')) false end @@ -2254,6 +2270,7 @@ class Project < ApplicationRecord .concat(dependency_proxy_variables) .concat(auto_devops_variables) .concat(api_variables) + .concat(ci_template_variables) end end @@ -2307,6 +2324,12 @@ class Project < ApplicationRecord end end + def ci_template_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_TEMPLATE_REGISTRY_HOST', value: 'registry.gitlab.com') + end + end + def dependency_proxy_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless Gitlab.config.dependency_proxy.enabled @@ -2651,7 +2674,7 @@ class Project < ApplicationRecord { repository_storage: repository_storage, - pool_repository: pool_repository || create_new_pool_repository + pool_repository: pool_repository || create_new_pool_repository } end @@ -2880,6 +2903,12 @@ class Project < ApplicationRecord ci_cd_settings.forward_deployment_enabled? end + def ci_allow_fork_pipelines_to_run_in_parent_project? + return false unless ci_cd_settings + + ci_cd_settings.allow_fork_pipelines_to_run_in_parent_project? + end + def ci_job_token_scope_enabled? return false unless ci_cd_settings @@ -2984,6 +3013,14 @@ class Project < ApplicationRecord group&.work_items_feature_flag_enabled? || Feature.enabled?(:work_items, self) end + def work_items_mvc_2_feature_flag_enabled? + group&.work_items_mvc_2_feature_flag_enabled? || Feature.enabled?(:work_items_mvc_2) + end + + def work_items_create_from_markdown_feature_flag_enabled? + work_items_feature_flag_enabled? && (group&.work_items_create_from_markdown_feature_flag_enabled? || Feature.enabled?(:work_items_create_from_markdown)) + end + def enqueue_record_project_target_platforms return unless Gitlab.com? return unless Feature.enabled?(:record_projects_target_platforms, self) @@ -3008,6 +3045,10 @@ class Project < ApplicationRecord licensed_feature_available?(:security_training) end + def destroy_deployment_by_id(deployment_id) + deployments.where(id: deployment_id).fast_destroy_all + end + private # overridden in EE @@ -3238,6 +3279,12 @@ class Project < ApplicationRecord project_namespace.assign_attributes(attributes_to_sync) end + def reload_project_namespace_details + return unless (previous_changes.keys & %w(description description_html cached_markdown_version)).any? && project_namespace.namespace_details.present? + + project_namespace.namespace_details.reset + end + # SyncEvents are created by PG triggers (with the function `insert_projects_sync_event`) def schedule_sync_event_worker run_after_commit do diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 0a30e125c83..8623e477c06 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -21,6 +21,9 @@ class ProjectFeature < ApplicationRecord security_and_compliance container_registry package_registry + environments + feature_flags + releases ].freeze EXPORTABLE_FEATURES = (FEATURES - [:security_and_compliance, :pages]).freeze diff --git a/app/models/projects/import_export/relation_export.rb b/app/models/projects/import_export/relation_export.rb index 0a31e525ac2..15198049f87 100644 --- a/app/models/projects/import_export/relation_export.rb +++ b/app/models/projects/import_export/relation_export.rb @@ -3,6 +3,20 @@ module Projects module ImportExport class RelationExport < ApplicationRecord + DESIGN_REPOSITORY_RELATION = 'design_repository' + LFS_OBJECTS_RELATION = 'lfs_objects' + REPOSITORY_RELATION = 'repository' + ROOT_RELATION = 'project' + SNIPPETS_REPOSITORY_RELATION = 'snippets_repository' + UPLOADS_RELATION = 'uploads' + WIKI_REPOSITORY_RELATION = 'wiki_repository' + + EXTRA_RELATION_LIST = [ + DESIGN_REPOSITORY_RELATION, LFS_OBJECTS_RELATION, REPOSITORY_RELATION, ROOT_RELATION, + SNIPPETS_REPOSITORY_RELATION, UPLOADS_RELATION, WIKI_REPOSITORY_RELATION + ].freeze + private_constant :EXTRA_RELATION_LIST + self.table_name = 'project_relation_exports' belongs_to :project_export_job @@ -17,6 +31,33 @@ module Projects validates :project_export_job, presence: true validates :relation, presence: true, length: { maximum: 255 }, uniqueness: { scope: :project_export_job_id } validates :status, numericality: { only_integer: true }, presence: true + + scope :by_relation, -> (relation) { where(relation: relation) } + + state_machine :status, initial: :queued do + state :queued, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :failed, value: 3 + + event :start do + transition queued: :started + end + + event :finish do + transition started: :finished + end + + event :fail_op do + transition [:queued, :started] => :failed + end + end + + def self.relation_names_list + project_tree_relation_names = ::Gitlab::ImportExport::Reader.new(shared: nil).project_relation_names.map(&:to_s) + + project_tree_relation_names + EXTRA_RELATION_LIST + end end end end diff --git a/app/models/projects/topic.rb b/app/models/projects/topic.rb index bc7f94e4374..b0f138714a0 100644 --- a/app/models/projects/topic.rb +++ b/app/models/projects/topic.rb @@ -15,6 +15,7 @@ module Projects has_many :project_topics, class_name: 'Projects::ProjectTopic' has_many :projects, through: :project_topics + scope :without_assigned_projects, -> { where(total_projects_count: 0) } scope :order_by_non_private_projects_count, -> { order(non_private_projects_count: :desc).order(id: :asc) } scope :reorder_by_similarity, -> (search) do order_expression = Gitlab::Database::SimilarityScore.build_expression(search: search, rules: [ diff --git a/app/models/prometheus_alert.rb b/app/models/prometheus_alert.rb index 684f50d5f58..9080f3d9de1 100644 --- a/app/models/prometheus_alert.rb +++ b/app/models/prometheus_alert.rb @@ -25,7 +25,7 @@ class PrometheusAlert < ApplicationRecord validates :environment, :project, :prometheus_metric, :threshold, :operator, presence: true validates :runbook_url, length: { maximum: 255 }, allow_blank: true, - addressable_url: { enforce_sanitization: true, ascii_only: true } + addressable_url: { enforce_sanitization: true, ascii_only: true } validate :require_valid_environment_project! validate :require_valid_metric_project! diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 7cf15439b47..76c277e4b86 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -4,8 +4,6 @@ class ProtectedBranch < ApplicationRecord include ProtectedRef include Gitlab::SQL::Pattern - CACHE_EXPIRE_IN = 1.hour - scope :requiring_code_owner_approval, -> { where(code_owner_approval_required: true) } @@ -27,10 +25,30 @@ class ProtectedBranch < ApplicationRecord end # Check if branch name is marked as protected in the system - def self.protected?(project, ref_name) + def self.protected?(project, ref_name, dry_run: true) return true if project.empty_repo? && project.default_branch_protected? return false if ref_name.blank? + new_cache_result = new_cache(project, ref_name, dry_run: dry_run) + + return new_cache_result unless new_cache_result.nil? + + deprecated_cache(project, ref_name) + end + + def self.new_cache(project, ref_name, dry_run: true) + if Feature.enabled?(:hash_based_cache_for_protected_branches, project) + ProtectedBranches::CacheService.new(project).fetch(ref_name, dry_run: dry_run) do # rubocop: disable CodeReuse/ServiceClass + self.matching(ref_name, protected_refs: protected_refs(project)).present? + end + end + end + + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/368279 + # ---------------------------------------------------------------- + CACHE_EXPIRE_IN = 1.hour + + def self.deprecated_cache(project, ref_name) Rails.cache.fetch(protected_ref_cache_key(project, ref_name), expires_in: CACHE_EXPIRE_IN) do self.matching(ref_name, protected_refs: protected_refs(project)).present? end @@ -39,6 +57,7 @@ class ProtectedBranch < ApplicationRecord def self.protected_ref_cache_key(project, ref_name) "protected_ref-#{project.cache_key}-#{Digest::SHA1.hexdigest(ref_name)}" end + # End of deprecation -------------------------------------------- def self.allow_force_push?(project, ref_name) project.protected_branches.allowing_force_push.matching(ref_name).any? diff --git a/app/models/release.rb b/app/models/release.rb index ee5d7bab190..5ef3ff1bc6c 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -94,7 +94,7 @@ class Release < ApplicationRecord end def milestone_titles - self.milestones.order_by_dates_and_title.map {|m| m.title }.join(', ') + self.milestones.order_by_dates_and_title.map { |m| m.title }.join(', ') end def to_hook_data(action) diff --git a/app/models/release_highlight.rb b/app/models/release_highlight.rb index 17a9ad7db66..c2d498ecb13 100644 --- a/app/models/release_highlight.rb +++ b/app/models/release_highlight.rb @@ -33,7 +33,7 @@ class ReleaseHighlight next unless include_item?(item) begin - item.tap {|i| i['body'] = Banzai.render(i['body'], { project: nil }) } + item.tap { |i| i['description'] = Banzai.render(i['description'], { project: nil }) } rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, file_path: file_path) @@ -116,6 +116,6 @@ class ReleaseHighlight return true unless Gitlab::CurrentSettings.current_application_settings.whats_new_variant_current_tier? - item['packages']&.include?(current_package) + item['available_in']&.include?(current_package) end end diff --git a/app/models/repository.rb b/app/models/repository.rb index 9039bdf1a20..eb8e45877f3 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -244,10 +244,10 @@ class Repository end end - def add_branch(user, branch_name, ref) + def add_branch(user, branch_name, ref, expire_cache: true) branch = raw_repository.add_branch(branch_name, user: user, target: ref) - after_create_branch + after_create_branch(expire_cache: expire_cache) branch rescue Gitlab::Git::Repository::InvalidRef @@ -337,11 +337,17 @@ class Repository def expire_branches_cache expire_method_caches(%i(branch_names merged_branch_names branch_count has_visible_content? has_ambiguous_refs?)) + expire_protected_branches_cache + @local_branches = nil @branch_exists_memo = nil @branch_names_include = nil end + def expire_protected_branches_cache + ProtectedBranches::CacheService.new(project).refresh if project # rubocop:disable CodeReuse/ServiceClass + end + def expire_statistics_caches expire_method_caches(%i(size commit_count)) end @@ -646,8 +652,8 @@ class Repository return if licensee_object.name.blank? licensee_object - rescue Licensee::InvalidLicense => ex - Gitlab::ErrorTracking.track_exception(ex) + rescue Licensee::InvalidLicense => e + Gitlab::ErrorTracking.track_exception(e) nil end memoize_method :license @@ -1072,9 +1078,9 @@ class Repository ) do |commit_id| merge_request.update!(rebase_commit_sha: commit_id, merge_error: nil) end - rescue StandardError => error + rescue StandardError => e merge_request.update!(rebase_commit_sha: nil) - raise error + raise e end def squash(user, merge_request, message) diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index 5d7b3879d75..8fea0d6d993 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -68,7 +68,11 @@ class SentNotification < ApplicationRecord def noteable if for_commit? - project.commit(commit_id) rescue nil + begin + project.commit(commit_id) + rescue StandardError + nil + end else super end @@ -76,7 +80,11 @@ class SentNotification < ApplicationRecord def position=(new_position) if new_position.is_a?(String) - new_position = Gitlab::Json.parse(new_position) rescue nil + new_position = begin + Gitlab::Json.parse(new_position) + rescue StandardError + nil + end end if new_position.is_a?(Hash) diff --git a/app/models/serverless/domain_cluster.rb b/app/models/serverless/domain_cluster.rb index 0d54a97370e..1effabf1c22 100644 --- a/app/models/serverless/domain_cluster.rb +++ b/app/models/serverless/domain_cluster.rb @@ -17,7 +17,7 @@ module Serverless validates :pages_domain, :knative, presence: true validates :uuid, presence: true, uniqueness: true, length: { is: ::Serverless::Domain::UUID_LENGTH }, - format: { with: HEX_REGEXP, message: 'only allows hex characters' } + format: { with: HEX_REGEXP, message: 'only allows hex characters' } default_value_for(:uuid, allows_nil: false) { ::Serverless::Domain.generate_uuid } diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 47b23bbd28a..fd882633a44 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -94,8 +94,8 @@ class Snippet < ApplicationRecord attr_spammable :content, spam_description: true attr_encrypted :secret_token, - key: Settings.attr_encrypted_db_key_base_truncated, - mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_truncated, + mode: :per_attribute_iv, algorithm: 'aes-256-cbc' class << self diff --git a/app/models/snippet_repository.rb b/app/models/snippet_repository.rb index 92405a0d943..5ac159d9615 100644 --- a/app/models/snippet_repository.rb +++ b/app/models/snippet_repository.rb @@ -44,11 +44,11 @@ class SnippetRepository < ApplicationRecord Gitlab::Git::CommitError, Gitlab::Git::PreReceiveError, Gitlab::Git::CommandError, - ArgumentError => error + ArgumentError => e - logger.error(message: "Snippet git error. Reason: #{error.message}", snippet: snippet.id) + logger.error(message: "Snippet git error. Reason: #{e.message}", snippet: snippet.id) - raise commit_error_exception(error) + raise commit_error_exception(e) end def transform_file_entries(files) diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 2643ef272d8..cc389dbe3f4 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -22,7 +22,7 @@ class SystemNoteMetadata < ApplicationRecord designs_added designs_modified designs_removed designs_discussion_added title time_tracking branch milestone discussion task moved cloned opened closed merged duplicate locked unlocked outdated reviewer - tag due_date pinned_embed cherry_pick health_status approved unapproved + tag due_date start_date_or_due_date pinned_embed cherry_pick health_status approved unapproved status alert_issue_added relate unrelate new_alert_added severity attention_requested attention_request_removed contact timeline_event ].freeze diff --git a/app/models/terraform/state.rb b/app/models/terraform/state.rb index 59f7d852ce6..e5c8f4ab32a 100644 --- a/app/models/terraform/state.rb +++ b/app/models/terraform/state.rb @@ -26,7 +26,7 @@ module Terraform validates :project_id, :name, presence: true validates :uuid, presence: true, uniqueness: true, length: { is: UUID_LENGTH }, - format: { with: HEX_REGEXP, message: 'only allows hex characters' } + format: { with: HEX_REGEXP, message: 'only allows hex characters' } default_value_for(:uuid, allows_nil: false) { SecureRandom.hex(UUID_LENGTH / 2) } diff --git a/app/models/todo.rb b/app/models/todo.rb index c698783d750..d165e60e4c3 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -19,7 +19,6 @@ class Todo < ApplicationRecord DIRECTLY_ADDRESSED = 7 MERGE_TRAIN_REMOVED = 8 # This is an EE-only feature REVIEW_REQUESTED = 9 - ATTENTION_REQUESTED = 10 ACTION_NAMES = { ASSIGNED => :assigned, @@ -30,8 +29,7 @@ class Todo < ApplicationRecord APPROVAL_REQUIRED => :approval_required, UNMERGEABLE => :unmergeable, DIRECTLY_ADDRESSED => :directly_addressed, - MERGE_TRAIN_REMOVED => :merge_train_removed, - ATTENTION_REQUESTED => :attention_requested + MERGE_TRAIN_REMOVED => :merge_train_removed }.freeze ACTIONS_MULTIPLE_ALLOWED = [Todo::MENTIONED, Todo::DIRECTLY_ADDRESSED].freeze @@ -195,10 +193,6 @@ class Todo < ApplicationRecord action == REVIEW_REQUESTED end - def attention_requested? - action == ATTENTION_REQUESTED - end - def merge_train_removed? action == MERGE_TRAIN_REMOVED end @@ -238,7 +232,11 @@ class Todo < ApplicationRecord # override to return commits, which are not active record def target if for_commit? - project.commit(commit_id) rescue nil + begin + project.commit(commit_id) + rescue StandardError + nil + end else super end diff --git a/app/models/u2f_registration.rb b/app/models/u2f_registration.rb index 7c01aa7a420..ba6c1ee6af1 100644 --- a/app/models/u2f_registration.rb +++ b/app/models/u2f_registration.rb @@ -6,21 +6,7 @@ class U2fRegistration < ApplicationRecord belongs_to :user after_create :create_webauthn_registration - after_update :update_webauthn_registration, if: :counter_changed? - - def create_webauthn_registration - converter = Gitlab::Auth::U2fWebauthnConverter.new(self) - WebauthnRegistration.create!(converter.convert) - rescue StandardError => ex - Gitlab::ErrorTracking.track_exception(ex, u2f_registration_id: self.id) - end - - def update_webauthn_registration - # When we update the sign count of this registration - # we need to update the sign count of the corresponding webauthn registration - # as well if it exists already - WebauthnRegistration.find_by_credential_xid(webauthn_credential_xid)&.update_attribute(:counter, counter) - end + after_update :update_webauthn_registration, if: :saved_change_to_counter? def self.register(user, app_id, params, challenges) u2f = U2F::U2F.new(app_id) @@ -60,10 +46,22 @@ class U2fRegistration < ApplicationRecord private + def create_webauthn_registration + converter = Gitlab::Auth::U2fWebauthnConverter.new(self) + WebauthnRegistration.create!(converter.convert) + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, u2f_registration_id: self.id) + end + + def update_webauthn_registration + # When we update the sign count of this registration + # we need to update the sign count of the corresponding webauthn registration + # as well if it exists already + WebauthnRegistration.find_by_credential_xid(webauthn_credential_xid) + &.update_attribute(:counter, counter) + end + def webauthn_credential_xid - # To find the corresponding webauthn registration, we use that - # the key handle of the u2f reg corresponds to the credential xid of the webauthn reg - # (with some base64 back and forth) Base64.strict_encode64(Base64.urlsafe_decode64(key_handle)) end end diff --git a/app/models/user.rb b/app/models/user.rb index 188b27383f9..afee2d70844 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -30,6 +30,7 @@ class User < ApplicationRecord include Gitlab::Auth::Otp::Fortinet include RestrictedSignup include StripAttribute + include EachBatch DEFAULT_NOTIFICATION_LEVEL = :participating @@ -69,8 +70,8 @@ class User < ApplicationRecord default_value_for :theme_id, gitlab_config.default_theme attr_encrypted :otp_secret, - key: Gitlab::Application.secrets.otp_key_base, - mode: :per_attribute_iv_and_salt, + key: Gitlab::Application.secrets.otp_key_base, + mode: :per_attribute_iv_and_salt, insecure_mode: true, algorithm: 'aes-256-cbc' @@ -222,6 +223,7 @@ class User < ApplicationRecord has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'Users::Callout' has_many :group_callouts, class_name: 'Users::GroupCallout' + has_many :project_callouts, class_name: 'Users::ProjectCallout' has_many :namespace_callouts, class_name: 'Users::NamespaceCallout' has_many :term_agreements belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' @@ -272,10 +274,10 @@ class User < ApplicationRecord validate :check_username_format, if: :username_changed? validates :theme_id, allow_nil: true, inclusion: { in: Gitlab::Themes.valid_ids, - message: _("%{placeholder} is not a valid theme") % { placeholder: '%{value}' } } + message: _("%{placeholder} is not a valid theme") % { placeholder: '%{value}' } } validates :color_scheme_id, allow_nil: true, inclusion: { in: Gitlab::ColorSchemes.valid_ids, - message: _("%{placeholder} is not a valid color scheme") % { placeholder: '%{value}' } } + message: _("%{placeholder} is not a valid color scheme") % { placeholder: '%{value}' } } validates :website_url, allow_blank: true, url: true, if: :website_url_changed? @@ -447,6 +449,11 @@ class User < ApplicationRecord scope :without_projects, -> { joins('LEFT JOIN project_authorizations ON users.id = project_authorizations.user_id').where(project_authorizations: { user_id: nil }) } scope :by_username, -> (usernames) { iwhere(username: Array(usernames).map(&:to_s)) } scope :by_name, -> (names) { iwhere(name: Array(names)) } + scope :by_login, -> (login) do + return none if login.blank? + + login.include?('@') ? iwhere(email: login) : iwhere(username: login) + end scope :by_user_email, -> (emails) { iwhere(email: Array(emails)) } scope :by_emails, -> (emails) { joins(:emails).where(emails: { email: Array(emails).map(&:downcase) }) } scope :for_todos, -> (todos) { where(id: todos.select(:user_id).distinct) } @@ -481,7 +488,6 @@ class User < ApplicationRecord scope :order_oldest_sign_in, -> { reorder(arel_table[:current_sign_in_at].asc.nulls_last) } scope :order_recent_last_activity, -> { reorder(arel_table[:last_activity_on].desc.nulls_last, arel_table[:id].asc) } scope :order_oldest_last_activity, -> { reorder(arel_table[:last_activity_on].asc.nulls_first, arel_table[:id].desc) } - scope :by_id_and_login, ->(id, login) { where(id: id).where('username = LOWER(:login) OR email = LOWER(:login)', login: login) } scope :dormant, -> { with_state(:active).human_or_service_user.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) } scope :with_no_activity, -> { with_state(:active).human_or_service_user.where(last_activity_on: nil).where('created_at <= ?', MINIMUM_DAYS_CREATED.day.ago.to_date) } scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) } @@ -691,33 +697,29 @@ class User < ApplicationRecord scope = options[:with_private_emails] ? with_primary_or_secondary_email(query) : with_public_email(query) scope = scope.or(search_by_name_or_username(query, use_minimum_char_limit: options[:use_minimum_char_limit])) - if Feature.enabled?(:use_keyset_aware_user_search_query) - order = Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'users_match_priority', - order_expression: sanitized_order_sql.asc, - add_to_projections: true, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'users_name', - order_expression: arel_table[:name].asc, - add_to_projections: true, - nullable: :not_nullable, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'users_id', - order_expression: arel_table[:id].asc, - add_to_projections: true, - nullable: :not_nullable, - distinct: true - ) - ]) - scope.reorder(order) - else - scope.reorder(sanitized_order_sql, :name) - end + order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'users_match_priority', + order_expression: sanitized_order_sql.asc, + add_to_projections: true, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'users_name', + order_expression: arel_table[:name].asc, + add_to_projections: true, + nullable: :not_nullable, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'users_id', + order_expression: arel_table[:id].asc, + add_to_projections: true, + nullable: :not_nullable, + distinct: true + ) + ]) + scope.reorder(order) end # Limits the result set to users _not_ in the given query/list of IDs. @@ -768,14 +770,8 @@ class User < ApplicationRecord true end - def by_login(login) - return unless login - - if login.include?('@') - unscoped.iwhere(email: login).take - else - unscoped.iwhere(username: login).take - end + def find_by_login(login) + by_login(login).take end def find_by_username(username) @@ -991,12 +987,12 @@ class User < ApplicationRecord def disable_two_factor! transaction do update( - otp_required_for_login: false, - encrypted_otp_secret: nil, - encrypted_otp_secret_iv: nil, - encrypted_otp_secret_salt: nil, + otp_required_for_login: false, + encrypted_otp_secret: nil, + encrypted_otp_secret_iv: nil, + encrypted_otp_secret_salt: nil, otp_grace_period_started_at: nil, - otp_backup_codes: nil + otp_backup_codes: nil ) self.u2f_registrations.destroy_all # rubocop: disable Cop/DestroyAll self.webauthn_registrations.destroy_all # rubocop: disable Cop/DestroyAll @@ -1663,7 +1659,14 @@ class User < ApplicationRecord end def forkable_namespaces - @forkable_namespaces ||= [namespace] + manageable_groups(include_groups_with_developer_maintainer_access: true) + strong_memoize(:forkable_namespaces) do + personal_namespace = Namespace.where(id: namespace_id) + + Namespace.from_union([ + manageable_groups(include_groups_with_developer_maintainer_access: true), + personal_namespace + ]) + end end def manageable_groups(include_groups_with_developer_maintainer_access: false) @@ -1808,16 +1811,6 @@ class User < ApplicationRecord end end - def attention_requested_open_merge_requests_count(force: false) - if Feature.enabled?(:uncached_mr_attention_requests_count, self) - MergeRequestsFinder.new(self, attention: self.username, state: 'opened', non_archived: true).execute.count - else - Rails.cache.fetch(attention_request_cache_key, force: force, expires_in: COUNT_CACHE_VALIDITY_PERIOD) do - MergeRequestsFinder.new(self, attention: self.username, state: 'opened', non_archived: true).execute.count - end - end - end - def assigned_open_issues_count(force: false) Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force, expires_in: COUNT_CACHE_VALIDITY_PERIOD) do IssuesFinder.new(self, assignee_id: self.id, state: 'opened', non_archived: true).execute.count @@ -1861,11 +1854,6 @@ class User < ApplicationRecord def invalidate_merge_request_cache_counts Rails.cache.delete(['users', id, 'assigned_open_merge_requests_count']) Rails.cache.delete(['users', id, 'review_requested_open_merge_requests_count']) - invalidate_attention_requested_count - end - - def invalidate_attention_requested_count - Rails.cache.delete(attention_request_cache_key) end def invalidate_todos_cache_counts @@ -1877,10 +1865,6 @@ class User < ApplicationRecord Rails.cache.delete(['users', id, 'personal_projects_count']) end - def attention_request_cache_key - ['users', id, 'attention_requested_open_merge_requests_count'] - end - # This is copied from Devise::Models::Lockable#valid_for_authentication?, as our auth # flow means we don't call that automatically (and can't conveniently do so). # @@ -2095,6 +2079,12 @@ class User < ApplicationRecord callout_dismissed?(callout, ignore_dismissal_earlier_than) end + def dismissed_callout_for_project?(feature_name:, project:, ignore_dismissal_earlier_than: nil) + callout = project_callouts.find_by(feature_name: feature_name, project: project) + + callout_dismissed?(callout, ignore_dismissal_earlier_than) + end + # Load the current highest access by looking directly at the user's memberships def current_highest_access_level members.non_request.maximum(:access_level) @@ -2126,6 +2116,11 @@ class User < ApplicationRecord .find_or_initialize_by(feature_name: ::Users::NamespaceCallout.feature_names[feature_name], namespace_id: namespace_id) end + def find_or_initialize_project_callout(feature_name, project_id) + project_callouts + .find_or_initialize_by(feature_name: ::Users::ProjectCallout.feature_names[feature_name], project_id: project_id) + end + def can_trigger_notifications? confirmed? && !blocked? && !ghost? end @@ -2160,6 +2155,10 @@ class User < ApplicationRecord Feature.enabled?(:mr_attention_requests, self) end + def account_age_in_days + (Date.current - created_at.to_date).to_i + end + protected # override, from Devise::Validatable diff --git a/app/models/user_status.rb b/app/models/user_status.rb index 7a803e8f1f6..dee976a4497 100644 --- a/app/models/user_status.rb +++ b/app/models/user_status.rb @@ -9,12 +9,12 @@ class UserStatus < ApplicationRecord CLEAR_STATUS_QUICK_OPTIONS = { '30_minutes' => 30.minutes, - '3_hours' => 3.hours, - '8_hours' => 8.hours, - '1_day' => 1.day, - '3_days' => 3.days, - '7_days' => 7.days, - '30_days' => 30.days + '3_hours' => 3.hours, + '8_hours' => 8.hours, + '1_day' => 1.day, + '3_days' => 3.days, + '7_days' => 7.days, + '30_days' => 30.days }.freeze belongs_to :user @@ -32,6 +32,10 @@ class UserStatus < ApplicationRecord def clear_status_after=(value) self.clear_status_at = CLEAR_STATUS_QUICK_OPTIONS[value]&.from_now end + + def customized? + message.present? || emoji != UserStatus::DEFAULT_EMOJI + end end UserStatus.prepend_mod_with('UserStatus') diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 570e3ae9b3c..7b5c7fef7ba 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -55,8 +55,13 @@ module Users preview_user_over_limit_free_plan_alert: 50, # EE-only user_reached_limit_free_plan_alert: 51, # EE-only submit_license_usage_data_banner: 52, # EE-only - personal_project_limitations_banner: 53, # EE-only - mr_experience_survey: 54 + personal_project_limitations_banner: 53, # EE-only + mr_experience_survey: 54, + namespace_storage_limit_banner_info_threshold: 55, # EE-only + namespace_storage_limit_banner_warning_threshold: 56, # EE-only + namespace_storage_limit_banner_alert_threshold: 57, # EE-only + namespace_storage_limit_banner_error_threshold: 58, # EE-only + project_quality_summary_feedback: 59 # EE-only } validates :feature_name, diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb index 0ea7b8199aa..70498ae83e0 100644 --- a/app/models/users/group_callout.rb +++ b/app/models/users/group_callout.rb @@ -17,7 +17,13 @@ module Users storage_enforcement_banner_fourth_enforcement_threshold: 6, preview_user_over_limit_free_plan_alert: 7, # EE-only user_reached_limit_free_plan_alert: 8, # EE-only - free_group_limited_alert: 9 # EE-only + free_group_limited_alert: 9, # EE-only + namespace_storage_limit_banner_info_threshold: 10, # EE-only + namespace_storage_limit_banner_warning_threshold: 11, # EE-only + namespace_storage_limit_banner_alert_threshold: 12, # EE-only + namespace_storage_limit_banner_error_threshold: 13, # EE-only + usage_quota_trial_alert: 14, # EE-only + preview_usage_quota_free_plan_alert: 15 # EE-only } validates :group, presence: true diff --git a/app/models/users/project_callout.rb b/app/models/users/project_callout.rb new file mode 100644 index 00000000000..ddc5f8fb4de --- /dev/null +++ b/app/models/users/project_callout.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Users + class ProjectCallout < ApplicationRecord + include Users::Calloutable + + self.table_name = 'user_project_callouts' + + belongs_to :project + + enum feature_name: { + awaiting_members_banner: 1 # EE-only + } + + validates :project, presence: true + validates :feature_name, + presence: true, + uniqueness: { scope: [:user_id, :project_id] }, + inclusion: { in: ProjectCallout.feature_names.keys } + end +end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index c9cb3b0b796..d28a73b644f 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -146,8 +146,8 @@ class Wiki repository.create_if_not_exists(default_branch) raise CouldNotCreateWikiError unless repository_exists? - rescue StandardError => err - Gitlab::ErrorTracking.track_exception(err, wiki: { + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, wiki: { container_type: container.class.name, container_id: container.id, full_path: full_path, @@ -335,7 +335,7 @@ class Wiki end def wiki_base_path - web_url(only_path: true).sub(%r{/#{Wiki::HOMEPAGE}\z}, '') + web_url(only_path: true).sub(%r{/#{Wiki::HOMEPAGE}\z}o, '') end # Callbacks for synchronous processing after wiki changes. @@ -364,9 +364,9 @@ class Wiki Gitlab::Git::CommitError, Gitlab::Git::PreReceiveError, Gitlab::Git::CommandError, - ArgumentError => error + ArgumentError => e - Gitlab::ErrorTracking.log_exception(error, action: action, wiki_id: id) + Gitlab::ErrorTracking.log_exception(e, action: action, wiki_id: id) false end diff --git a/app/models/work_item.rb b/app/models/work_item.rb index d29df0c31fc..451359c1f85 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -12,7 +12,7 @@ class WorkItem < Issue has_many :child_links, class_name: '::WorkItems::ParentLink', foreign_key: :work_item_parent_id has_many :work_item_children, through: :child_links, class_name: 'WorkItem', - foreign_key: :work_item_id, source: :work_item + foreign_key: :work_item_id, source: :work_item scope :inc_relations_for_permission_check, -> { includes(:author, project: :project_feature) } @@ -34,9 +34,22 @@ class WorkItem < Issue private + override :parent_link_confidentiality + def parent_link_confidentiality + if confidential? && work_item_children.public_only.exists? + errors.add(:confidential, _('confidential parent can not be used if there are non-confidential children.')) + end + + if !confidential? && work_item_parent&.confidential? + errors.add(:confidential, _('associated parent is confidential and can not have non-confidential children.')) + end + end + def record_create_action super Gitlab::UsageDataCounters::WorkItemActivityUniqueCounter.track_work_item_created_action(author: author) end end + +WorkItem.prepend_mod diff --git a/app/models/work_items/parent_link.rb b/app/models/work_items/parent_link.rb index f5ebbfa59b8..13d6db3e08e 100644 --- a/app/models/work_items/parent_link.rb +++ b/app/models/work_items/parent_link.rb @@ -16,6 +16,20 @@ module WorkItems validate :validate_parent_type validate :validate_same_project validate :validate_max_children + validate :validate_confidentiality + + class << self + def has_public_children?(parent_id) + joins(:work_item).where(work_item_parent_id: parent_id, 'issues.confidential': false).exists? + end + + def has_confidential_parent?(id) + link = find_by_work_item_id(id) + return false unless link + + link.work_item_parent.confidential? + end + end private @@ -56,5 +70,14 @@ module WorkItems errors.add :work_item_parent, _('parent already has maximum number of children.') end end + + def validate_confidentiality + return unless work_item_parent && work_item + + if work_item_parent.confidential? && !work_item.confidential? + errors.add :work_item, _("cannot assign a non-confidential work item to a confidential "\ + "parent. Make the work item confidential and try again.") + end + end end end diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb index e38d0ae153a..753fcbcb8f9 100644 --- a/app/models/work_items/type.rb +++ b/app/models/work_items/type.rb @@ -13,21 +13,23 @@ module WorkItems # Base types need to exist on the DB on app startup # This constant is used by the DB seeder BASE_TYPES = { - issue: { name: 'Issue', icon_name: 'issue-type-issue', enum_value: 0 }, - incident: { name: 'Incident', icon_name: 'issue-type-incident', enum_value: 1 }, - test_case: { name: 'Test Case', icon_name: 'issue-type-test-case', enum_value: 2 }, ## EE-only + issue: { name: 'Issue', icon_name: 'issue-type-issue', enum_value: 0 }, + incident: { name: 'Incident', icon_name: 'issue-type-incident', enum_value: 1 }, + test_case: { name: 'Test Case', icon_name: 'issue-type-test-case', enum_value: 2 }, ## EE-only requirement: { name: 'Requirement', icon_name: 'issue-type-requirements', enum_value: 3 }, ## EE-only - task: { name: 'Task', icon_name: 'issue-type-task', enum_value: 4 } + task: { name: 'Task', icon_name: 'issue-type-task', enum_value: 4 } }.freeze WIDGETS_FOR_TYPE = { - issue: [Widgets::Assignees, Widgets::Description, Widgets::Hierarchy, Widgets::Weight], + issue: [Widgets::Assignees, Widgets::Labels, Widgets::Description, Widgets::Hierarchy, Widgets::StartAndDueDate], incident: [Widgets::Description, Widgets::Hierarchy], test_case: [Widgets::Description], requirement: [Widgets::Description], - task: [Widgets::Assignees, Widgets::Description, Widgets::Hierarchy, Widgets::Weight] + task: [Widgets::Assignees, Widgets::Labels, Widgets::Description, Widgets::Hierarchy, Widgets::StartAndDueDate] }.freeze + WI_TYPES_WITH_CREATED_HEADER = %w[issue incident].freeze + cache_markdown_field :description, pipeline: :single_line enum base_type: BASE_TYPES.transform_values { |value| value[:enum_value] } @@ -83,3 +85,5 @@ module WorkItems end end end + +WorkItems::Type.prepend_mod diff --git a/app/models/work_items/widgets/labels.rb b/app/models/work_items/widgets/labels.rb new file mode 100644 index 00000000000..4ad8319ffac --- /dev/null +++ b/app/models/work_items/widgets/labels.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class Labels < Base + delegate :labels, to: :work_item + delegate :allows_scoped_labels?, to: :work_item + end + end +end diff --git a/app/models/work_items/widgets/start_and_due_date.rb b/app/models/work_items/widgets/start_and_due_date.rb new file mode 100644 index 00000000000..0b828c5b5a9 --- /dev/null +++ b/app/models/work_items/widgets/start_and_due_date.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class StartAndDueDate < Base + delegate :start_date, :due_date, to: :work_item + end + end +end diff --git a/app/models/work_items/widgets/weight.rb b/app/models/work_items/widgets/weight.rb deleted file mode 100644 index f589378f307..00000000000 --- a/app/models/work_items/widgets/weight.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module WorkItems - module Widgets - class Weight < Base - delegate :weight, to: :work_item - end - end -end |