diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 12:26:25 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 12:26:25 +0000 |
| commit | a09983ae35713f5a2bbb100981116d31ce99826e (patch) | |
| tree | 2ee2af7bd104d57086db360a7e6d8c9d5d43667a /app/models | |
| parent | 18c5ab32b738c0b6ecb4d0df3994000482f34bd8 (diff) | |
| download | gitlab-ce-a09983ae35713f5a2bbb100981116d31ce99826e.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to 'app/models')
136 files changed, 2497 insertions, 635 deletions
diff --git a/app/models/active_session.rb b/app/models/active_session.rb index a23190cc8b3..be07c221f32 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -91,8 +91,11 @@ class ActiveSession key_names = session_ids.map { |session_id| key_name(user.id, session_id.public_id) } redis.srem(lookup_key_name(user.id), session_ids.map(&:public_id)) - redis.del(key_names) - redis.del(rack_session_keys(session_ids)) + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.del(key_names) + redis.del(rack_session_keys(session_ids)) + end end def self.cleanup(user) @@ -136,8 +139,10 @@ class ActiveSession session_keys = rack_session_keys(session_ids) session_keys.each_slice(SESSION_BATCH_SIZE).flat_map do |session_keys_batch| - redis.mget(session_keys_batch).compact.map do |raw_session| - load_raw_session(raw_session) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.mget(session_keys_batch).compact.map do |raw_session| + load_raw_session(raw_session) + end end end end @@ -178,7 +183,9 @@ class ActiveSession entry_keys = session_ids.map { |session_id| key_name(user_id, session_id) } - redis.mget(entry_keys) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.mget(entry_keys) + end end def self.active_session_entries(session_ids, user_id, redis) diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb index af60ddd6f9a..fb166fb56b7 100644 --- a/app/models/alert_management/alert.rb +++ b/app/models/alert_management/alert.rb @@ -10,6 +10,7 @@ module AlertManagement include Sortable include Noteable include Gitlab::SQL::Pattern + include Presentable STATUSES = { triggered: 0, @@ -25,8 +26,17 @@ module AlertManagement ignored: :ignore }.freeze + OPEN_STATUSES = [ + :triggered, + :acknowledged + ].freeze + + DETAILS_IGNORED_PARAMS = %w(start_time).freeze + belongs_to :project belongs_to :issue, optional: true + belongs_to :prometheus_alert, optional: true + belongs_to :environment, optional: true has_many :alert_assignees, inverse_of: :alert has_many :assignees, through: :alert_assignees @@ -50,8 +60,12 @@ module AlertManagement validates :severity, presence: true validates :status, presence: true validates :started_at, presence: true - validates :fingerprint, uniqueness: { scope: :project }, allow_blank: true - validate :hosts_length + validates :fingerprint, allow_blank: true, uniqueness: { + scope: :project, + conditions: -> { not_resolved }, + message: -> (object, data) { _('Cannot have multiple unresolved alerts') } + }, unless: :resolved? + validate :hosts_length enum severity: { critical: 0, @@ -108,15 +122,30 @@ module AlertManagement scope :for_iid, -> (iid) { where(iid: iid) } scope :for_status, -> (status) { where(status: status) } scope :for_fingerprint, -> (project, fingerprint) { where(project: project, fingerprint: fingerprint) } + scope :for_environment, -> (environment) { where(environment: environment) } scope :search, -> (query) { fuzzy_search(query, [:title, :description, :monitoring_tool, :service]) } + scope :open, -> { with_status(OPEN_STATUSES) } + scope :not_resolved, -> { where.not(status: STATUSES[:resolved]) } + scope :with_prometheus_alert, -> { includes(:prometheus_alert) } scope :order_start_time, -> (sort_order) { order(started_at: sort_order) } scope :order_end_time, -> (sort_order) { order(ended_at: sort_order) } scope :order_event_count, -> (sort_order) { order(events: sort_order) } - scope :order_severity, -> (sort_order) { order(severity: sort_order) } - scope :order_status, -> (sort_order) { order(status: sort_order) } + + # Ascending sort order sorts severity from less critical to more critical. + # Descending sort order sorts severity from more critical to less critical. + # https://gitlab.com/gitlab-org/gitlab/-/issues/221242#what-is-the-expected-correct-behavior + scope :order_severity, -> (sort_order) { order(severity: sort_order == :asc ? :desc : :asc) } + + # Ascending sort order sorts statuses: Ignored > Resolved > Acknowledged > Triggered + # Descending sort order sorts statuses: Triggered > Acknowledged > Resolved > Ignored + # https://gitlab.com/gitlab-org/gitlab/-/issues/221242#what-is-the-expected-correct-behavior + scope :order_status, -> (sort_order) { order(status: sort_order == :asc ? :desc : :asc) } scope :counts_by_status, -> { group(:status).count } + scope :counts_by_project_id, -> { group(:project_id).count } + + alias_method :state, :status_name def self.sort_by_attribute(method) case method.to_s @@ -135,8 +164,13 @@ module AlertManagement end end + def self.last_prometheus_alert_by_project_id + ids = select(arel_table[:id].maximum).group(:project_id) + with_prometheus_alert.where(id: ids) + end + def details - details_payload = payload.except(*attributes.keys) + details_payload = payload.except(*attributes.keys, *DETAILS_IGNORED_PARAMS) Gitlab::Utils::InlineHash.merge_keys(details_payload) end @@ -161,6 +195,12 @@ module AlertManagement project.execute_services(hook_data, :alert_hooks) end + def present + return super(presenter_class: AlertManagement::PrometheusAlertPresenter) if prometheus? + + super + end + private def hook_data diff --git a/app/models/application_record.rb b/app/models/application_record.rb index c7e4d64d3d5..9ec407a10a4 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -13,6 +13,10 @@ class ApplicationRecord < ActiveRecord::Base where(id: ids) end + def self.iid_in(iids) + where(iid: iids) + end + def self.id_not_in(ids) where.not(id: ids) end @@ -34,6 +38,10 @@ class ApplicationRecord < ActiveRecord::Base false end + def self.at_most(count) + limit(count) + end + def self.safe_find_or_create_by!(*args) safe_find_or_create_by(*args).tap do |record| record.validate! unless record.persisted? diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index d24136cc04a..c489d11d462 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -50,6 +50,7 @@ module ApplicationSettingImplementation default_artifacts_expire_in: '30 days', default_branch_protection: Settings.gitlab['default_branch_protection'], default_ci_config_path: nil, + default_branch_name: nil, default_group_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_project_creation: Settings.gitlab['default_project_creation'], default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], @@ -88,6 +89,7 @@ module ApplicationSettingImplementation max_attachment_size: Settings.gitlab['max_attachment_size'], max_import_size: 50, mirror_available: true, + notify_on_unknown_sign_in: true, outbound_local_requests_whitelist: [], password_authentication_enabled_for_git: true, password_authentication_enabled_for_web: Settings.gitlab['signin_enabled'], @@ -156,7 +158,13 @@ module ApplicationSettingImplementation snowplow_iglu_registry_url: nil, custom_http_clone_url_root: nil, productivity_analytics_start_date: Time.current, - snippet_size_limit: 50.megabytes + snippet_size_limit: 50.megabytes, + project_import_limit: 6, + project_export_limit: 6, + project_download_export_limit: 1, + group_import_limit: 6, + group_export_limit: 6, + group_download_export_limit: 1 } end diff --git a/app/models/approval.rb b/app/models/approval.rb new file mode 100644 index 00000000000..bc123de0b20 --- /dev/null +++ b/app/models/approval.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Approval < ApplicationRecord + belongs_to :user + belongs_to :merge_request + + validates :merge_request_id, presence: true + validates :user_id, presence: true, uniqueness: { scope: [:merge_request_id] } + + scope :with_user, -> { joins(:user) } +end diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 3bbd2e43a51..13fc2514f0c 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -3,8 +3,11 @@ class AuditEvent < ApplicationRecord include CreatedAtFilterable include IgnorableColumns + include BulkInsertSafe - ignore_column :updated_at, remove_with: '13.3', remove_after: '2020-08-22' + PARALLEL_PERSISTENCE_COLUMNS = [:author_name, :entity_path].freeze + + ignore_column :updated_at, remove_with: '13.4', remove_after: '2020-09-22' serialize :details, Hash # rubocop:disable Cop/ActiveRecordSerialize @@ -16,8 +19,15 @@ class AuditEvent < ApplicationRecord scope :by_entity_type, -> (entity_type) { where(entity_type: entity_type) } scope :by_entity_id, -> (entity_id) { where(entity_id: entity_id) } + scope :by_author_id, -> (author_id) { where(author_id: author_id) } after_initialize :initialize_details + # Note: The intention is to remove this once refactoring of AuditEvent + # has proceeded further. + # + # See further details in the epic: + # https://gitlab.com/groups/gitlab-org/-/epics/2765 + after_validation :parallel_persist def self.order_by(method) case method.to_s @@ -51,7 +61,11 @@ class AuditEvent < ApplicationRecord private def default_author_value - ::Gitlab::Audit::NullAuthor.for(author_id, details[:author_name]) + ::Gitlab::Audit::NullAuthor.for(author_id, (self[:author_name] || details[:author_name])) + end + + def parallel_persist + PARALLEL_PERSISTENCE_COLUMNS.each { |col| self[col] = details[col] } end end diff --git a/app/models/blob_viewer/image.rb b/app/models/blob_viewer/image.rb index cbebef46c60..97eb0489158 100644 --- a/app/models/blob_viewer/image.rb +++ b/app/models/blob_viewer/image.rb @@ -8,7 +8,7 @@ module BlobViewer self.partial_name = 'image' self.extensions = UploaderHelper::SAFE_IMAGE_EXT self.binary = true - self.switcher_icon = 'picture-o' + self.switcher_icon = 'doc-image' self.switcher_title = 'image' end end diff --git a/app/models/blob_viewer/notebook.rb b/app/models/blob_viewer/notebook.rb index 57d6d802db3..351502d451f 100644 --- a/app/models/blob_viewer/notebook.rb +++ b/app/models/blob_viewer/notebook.rb @@ -8,7 +8,7 @@ module BlobViewer self.partial_name = 'notebook' self.extensions = %w(ipynb) self.binary = false - self.switcher_icon = 'file-text-o' + self.switcher_icon = 'doc-text' self.switcher_title = 'notebook' end end diff --git a/app/models/blob_viewer/open_api.rb b/app/models/blob_viewer/open_api.rb index 963b7336c8d..0551f3bb1e3 100644 --- a/app/models/blob_viewer/open_api.rb +++ b/app/models/blob_viewer/open_api.rb @@ -8,8 +8,6 @@ module BlobViewer self.partial_name = 'openapi' self.file_types = %i(openapi) self.binary = false - # TODO: get an icon for OpenAPI - self.switcher_icon = 'file-pdf-o' - self.switcher_title = 'OpenAPI' + self.switcher_icon = 'api' end end diff --git a/app/models/blob_viewer/rich.rb b/app/models/blob_viewer/rich.rb index 0f66a672102..46f36cc2674 100644 --- a/app/models/blob_viewer/rich.rb +++ b/app/models/blob_viewer/rich.rb @@ -6,7 +6,7 @@ module BlobViewer included do self.type = :rich - self.switcher_icon = 'file-text-o' + self.switcher_icon = 'doc-text' self.switcher_title = 'rendered file' end end diff --git a/app/models/blob_viewer/svg.rb b/app/models/blob_viewer/svg.rb index 454c6a57568..60a11fbd97e 100644 --- a/app/models/blob_viewer/svg.rb +++ b/app/models/blob_viewer/svg.rb @@ -8,7 +8,7 @@ module BlobViewer self.partial_name = 'svg' self.extensions = %w(svg) self.binary = false - self.switcher_icon = 'picture-o' + self.switcher_icon = 'doc-image' self.switcher_title = 'image' end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b5e68b55f72..6c90645e997 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -27,7 +27,7 @@ module Ci upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? }, refspecs: -> (build) { build.merge_request_ref? }, artifacts_exclude: -> (build) { build.supports_artifacts_exclude? }, - release_steps: -> (build) { build.release_steps? } + multi_build_steps: -> (build) { build.multi_build_steps? } }.freeze DEFAULT_RETRIES = { @@ -539,7 +539,6 @@ module Ci .concat(job_variables) .concat(environment_changed_page_variables) .concat(persisted_environment_variables) - .concat(deploy_freeze_variables) .to_runner_variables end end @@ -595,18 +594,6 @@ module Ci end end - def deploy_freeze_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables unless freeze_period? - - variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') - end - end - - def freeze_period? - Ci::FreezePeriodStatus.new(project: project).execute - end - def dependency_variables return [] if all_dependencies.empty? @@ -801,6 +788,11 @@ module Ci has_expiring_artifacts? && job_artifacts_archive.present? end + def self.keep_artifacts! + update_all(artifacts_expire_at: nil) + Ci::JobArtifact.where(job: self.select(:id)).update_all(expire_at: nil) + end + def keep_artifacts! self.update(artifacts_expire_at: nil) self.job_artifacts.update_all(expire_at: nil) @@ -885,7 +877,7 @@ module Ci Gitlab::Ci::Features.artifacts_exclude_enabled? end - def release_steps? + def multi_build_steps? options.dig(:release)&.any? && Gitlab::Ci::Features.release_generation_enabled? end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 0df5ebfe843..4094bdb26dc 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -19,6 +19,7 @@ module Ci before_create :set_build_project validates :build, presence: true + validates :secrets, json_schema: { filename: 'build_metadata_secrets' } serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize serialize :config_variables, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index 0b243c20e67..b977a5f4419 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -4,6 +4,8 @@ module Ci class BuildNeed < ApplicationRecord extend Gitlab::Ci::Model + include BulkInsertSafe + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :needs validates :build, presence: true diff --git a/app/models/ci/build_trace.rb b/app/models/ci/build_trace.rb index b9db1559836..f70e1ed69ea 100644 --- a/app/models/ci/build_trace.rb +++ b/app/models/ci/build_trace.rb @@ -2,40 +2,22 @@ module Ci class BuildTrace - CONVERTERS = { - html: Gitlab::Ci::Ansi2html, - json: Gitlab::Ci::Ansi2json - }.freeze - attr_reader :trace, :build delegate :state, :append, :truncated, :offset, :size, :total, to: :trace, allow_nil: true delegate :id, :status, :complete?, to: :build, prefix: true - def initialize(build:, stream:, state:, content_format:) + def initialize(build:, stream:, state:) @build = build - @content_format = content_format if stream.valid? stream.limit - @trace = CONVERTERS.fetch(content_format).convert(stream.stream, state) + @trace = Gitlab::Ci::Ansi2json.convert(stream.stream, state) end end - def json? - @content_format == :json - end - - def html? - @content_format == :html - end - - def json_lines - @trace&.lines if json? - end - - def html_lines - @trace&.html if html? + def lines + @trace&.lines end end end diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb index 813eaf5d839..c3864f78b01 100644 --- a/app/models/ci/build_trace_chunks/redis.rb +++ b/app/models/ci/build_trace_chunks/redis.rb @@ -35,7 +35,10 @@ module Ci keys = keys.map { |key| key_raw(*key) } Gitlab::Redis::SharedState.with do |redis| - redis.del(keys) + # https://gitlab.com/gitlab-org/gitlab/-/issues/224171 + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.del(keys) + end end end diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb index 8245729a884..628749b32cb 100644 --- a/app/models/ci/instance_variable.rb +++ b/app/models/ci/instance_variable.rb @@ -45,13 +45,5 @@ module Ci end end end - - private - - def validate_plan_limit_not_exceeded - if Gitlab::Ci::Features.instance_level_variables_limit_enabled? - super - end - end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 8aba9356949..dbeba1ece31 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -7,10 +7,13 @@ module Ci include UpdateProjectStatistics include UsageStatistics include Sortable + include IgnorableColumns extend Gitlab::Ci::Model NotSupportedAdapterError = Class.new(StandardError) + ignore_columns :locked, remove_after: '2020-07-22', remove_with: '13.4' + TEST_REPORT_FILE_TYPES = %w[junit].freeze COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze @@ -34,13 +37,16 @@ module Ci license_management: 'gl-license-management-report.json', license_scanning: 'gl-license-scanning-report.json', performance: 'performance.json', + browser_performance: 'browser-performance.json', + load_performance: 'load-performance.json', metrics: 'metrics.txt', lsif: 'lsif.json', dotenv: '.env', cobertura: 'cobertura-coverage.xml', terraform: 'tfplan.json', cluster_applications: 'gl-cluster-applications.json', - requirements: 'requirements.json' + requirements: 'requirements.json', + coverage_fuzzing: 'gl-coverage-fuzzing.json' }.freeze INTERNAL_TYPES = { @@ -72,8 +78,11 @@ module Ci license_management: :raw, license_scanning: :raw, performance: :raw, + browser_performance: :raw, + load_performance: :raw, terraform: :raw, - requirements: :raw + requirements: :raw, + coverage_fuzzing: :raw }.freeze DOWNLOADABLE_TYPES = %w[ @@ -91,6 +100,8 @@ module Ci lsif metrics performance + browser_performance + load_performance sast secret_detection requirements @@ -98,9 +109,7 @@ module Ci TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze - # This is required since we cannot add a default to the database - # https://gitlab.com/gitlab-org/gitlab/-/issues/215418 - attribute :locked, :boolean, default: false + PLAN_LIMIT_PREFIX = 'ci_max_artifact_size_' belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id @@ -117,10 +126,9 @@ module Ci after_save :update_file_store, if: :saved_change_to_file? scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } - scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } + scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } - scope :for_ref, ->(ref, project_id) { joins(job: :pipeline).where(ci_pipelines: { ref: ref, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } scope :with_file_types, -> (file_types) do @@ -157,8 +165,7 @@ module Ci scope :expired, -> (limit) { where('expire_at < ?', Time.current).limit(limit) } scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } - scope :locked, -> { where(locked: true) } - scope :unlocked, -> { where(locked: [false, nil]) } + scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked).order(expire_at: :desc) } scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } @@ -176,7 +183,7 @@ module Ci codequality: 9, ## EE-specific license_management: 10, ## EE-specific license_scanning: 101, ## EE-specific till 13.0 - performance: 11, ## EE-specific + performance: 11, ## EE-specific till 13.2 metrics: 12, ## EE-specific metrics_referee: 13, ## runner referees network_referee: 14, ## runner referees @@ -187,7 +194,10 @@ module Ci accessibility: 19, cluster_applications: 20, secret_detection: 21, ## EE-specific - requirements: 22 ## EE-specific + requirements: 22, ## EE-specific + coverage_fuzzing: 23, ## EE-specific + browser_performance: 24, ## EE-specific + load_performance: 25 ## EE-specific } enum file_format: { @@ -235,6 +245,12 @@ module Ci self.update_column(:file_store, file.object_store) end + def self.associated_file_types_for(file_type) + return unless file_types.include?(file_type) + + [file_type] + end + def self.total_size self.sum(:size) end @@ -286,6 +302,21 @@ module Ci where(job_id: job_id).trace.take&.file&.file&.exists? end + def self.max_artifact_size(type:, project:) + max_size = if Feature.enabled?(:ci_max_artifact_size_per_type, project, default_enabled: false) + limit_name = "#{PLAN_LIMIT_PREFIX}#{type}" + + project.actual_limits.limit_for( + limit_name, + alternate_limit: -> { project.closest_setting(:max_artifacts_size) } + ) + else + project.closest_setting(:max_artifacts_size) + end + + max_size&.megabytes.to_i + end + private def file_format_adapter_class diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 497e1a4d74a..d4b439d648f 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -3,7 +3,7 @@ module Ci class Pipeline < ApplicationRecord extend Gitlab::Ci::Model - include HasStatus + include Ci::HasStatus include Importable include AfterCommitQueue include Presentable @@ -51,6 +51,8 @@ module Ci has_many :latest_builds, -> { latest }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' has_many :downloadable_artifacts, -> { not_expired.downloadable }, through: :latest_builds, source: :job_artifacts + has_many :messages, class_name: 'Ci::PipelineMessage', inverse_of: :pipeline + # Merge requests for which the current pipeline is running against # the merge request's latest commit. has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest' @@ -80,6 +82,7 @@ module Ci has_one :pipeline_config, class_name: 'Ci::PipelineConfig', inverse_of: :pipeline has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id + has_many :latest_builds_report_results, through: :latest_builds, source: :report_results accepts_nested_attributes_for :variables, reject_if: :persisted? @@ -110,6 +113,8 @@ module Ci # extend this `Hash` with new values. enum failure_reason: ::Ci::PipelineEnums.failure_reasons + enum locked: { unlocked: 0, artifacts_locked: 1 } + state_machine :status, initial: :created do event :enqueue do transition [:created, :manual, :waiting_for_resource, :preparing, :skipped, :scheduled] => :pending @@ -244,6 +249,14 @@ module Ci pipeline.run_after_commit { AutoDevops::DisableWorker.perform_async(pipeline.id) } end + + after_transition any => [:success] do |pipeline| + next unless Gitlab::Ci::Features.keep_latest_artifacts_for_ref_enabled?(pipeline.project) + + pipeline.run_after_commit do + Ci::PipelineSuccessUnlockArtifactsWorker.perform_async(pipeline.id) + end + end end scope :internal, -> { where(source: internal_sources) } @@ -256,7 +269,14 @@ module Ci scope :for_ref, -> (ref) { where(ref: ref) } scope :for_id, -> (id) { where(id: id) } scope :for_iid, -> (iid) { where(iid: iid) } + scope :for_project, -> (project) { where(project: project) } scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } + scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } + scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } + + scope :outside_pipeline_family, ->(pipeline) do + where.not(id: pipeline.same_family_pipeline_ids) + end scope :with_reports, -> (reports_scope) do where('EXISTS (?)', ::Ci::Build.latest.with_reports(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1)) @@ -270,6 +290,15 @@ module Ci ) end + # Returns the pipelines that associated with the given merge request. + # In general, please use `Ci::PipelinesForMergeRequestFinder` instead, + # for checking permission of the actor. + scope :triggered_by_merge_request, -> (merge_request) do + ci_sources.where(source: :merge_request_event, + merge_request: merge_request, + project: [merge_request.source_project, merge_request.target_project]) + end + # Returns the pipelines in descending order (= newest first), optionally # limited to a number of references. # @@ -348,6 +377,10 @@ module Ci success.group(:project_id).select('max(id) as id') end + def self.last_finished_for_ref_id(ci_ref_id) + where(ci_ref_id: ci_ref_id).ci_sources.finished.order(id: :desc).select(:id).take + end + def self.truncate_sha(sha) sha[0...8] end @@ -440,6 +473,10 @@ module Ci end end + def triggered_pipelines_with_preloads + triggered_pipelines.preload(:source_job) + end + def legacy_stages if ::Gitlab::Ci::Features.composite_status?(project) legacy_stages_using_composite_status @@ -552,10 +589,28 @@ module Ci end end + def lazy_ref_commit + return unless ::Gitlab::Ci::Features.pipeline_latest? + + BatchLoader.for(ref).batch do |refs, loader| + next unless project.repository_exists? + + project.repository.list_commits_by_ref_name(refs).then do |commits| + commits.each { |key, commit| loader.call(key, commits[key]) } + end + end + end + def latest? return false unless git_ref && commit.present? - project.commit(git_ref) == commit + unless ::Gitlab::Ci::Features.pipeline_latest? + return project.commit(git_ref) == commit + end + + return false if lazy_ref_commit.nil? + + lazy_ref_commit.id == commit.id end def retried @@ -569,10 +624,46 @@ module Ci end end + def batch_lookup_report_artifact_for_file_type(file_type) + latest_report_artifacts + .values_at(*::Ci::JobArtifact.associated_file_types_for(file_type.to_s)) + .flatten + .compact + .last + end + + # This batch loads the latest reports for each CI job artifact + # type (e.g. sast, dast, etc.) in a single SQL query to eliminate + # the need to do N different `job_artifacts.where(file_type: + # X).last` calls. + # + # Return a hash of file type => array of 1 job artifact + def latest_report_artifacts + ::Gitlab::SafeRequestStore.fetch("pipeline:#{self.id}:latest_report_artifacts") do + # Note we use read_attribute(:project_id) to read the project + # ID instead of self.project_id. The latter appears to load + # the Project model. This extra filter doesn't appear to + # affect query plan but included to ensure we don't leak the + # wrong informaiton. + ::Ci::JobArtifact.where( + id: job_artifacts.with_reports + .select('max(ci_job_artifacts.id) as id') + .where(project_id: self.read_attribute(:project_id)) + .group(:file_type) + ) + .preload(:job) + .group_by(&:file_type) + end + end + def has_kubernetes_active? project.deployment_platform&.active? end + def freeze_period? + Ci::FreezePeriodStatus.new(project: project).execute + end + def has_warnings? number_of_warnings.positive? end @@ -607,6 +698,25 @@ module Ci yaml_errors.present? end + def add_error_message(content) + add_message(:error, content) + end + + def add_warning_message(content) + add_message(:warning, content) + end + + # We can't use `messages.error` scope here because messages should also be + # read when the pipeline is not persisted. Using the scope will return no + # results as it would query persisted data. + def error_messages + messages.select(&:error?) + end + + def warning_messages + messages.select(&:warning?) + end + # Manually set the notes for a Ci::Pipeline # There is no ActiveRecord relation between Ci::Pipeline and notes # as they are related to a commit sha. This method helps importing @@ -639,7 +749,7 @@ module Ci when 'manual' then block when 'scheduled' then delay else - raise HasStatus::UnknownStatusError, + raise Ci::HasStatus::UnknownStatusError, "Unknown status `#{new_status}`" end end @@ -683,6 +793,7 @@ module Ci end variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if has_kubernetes_active? + variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if freeze_period? if external_pull_request_event? && external_pull_request variables.concat(external_pull_request.predefined_variables) @@ -748,13 +859,10 @@ module Ci end # If pipeline is a child of another pipeline, include the parent - # and the siblings, otherwise return only itself. + # and the siblings, otherwise return only itself and children. def same_family_pipeline_ids - if (parent = parent_pipeline) - [parent.id] + parent.child_pipelines.pluck(:id) - else - [self.id] - end + parent = parent_pipeline || self + [parent.id] + parent.child_pipelines.pluck(:id) end def bridge_triggered? @@ -802,6 +910,10 @@ module Ci complete? && latest_report_builds(reports_scope).exists? end + def test_report_summary + Gitlab::Ci::Reports::TestReportSummary.new(latest_builds_report_results) + end + def test_reports Gitlab::Ci::Reports::TestReports.new.tap do |test_reports| latest_report_builds(Ci::JobArtifact.test_reports).preload(:project).find_each do |build| @@ -840,6 +952,10 @@ module Ci end end + def has_archive_artifacts? + complete? && builds.latest.with_existing_job_artifacts(Ci::JobArtifact.archive.or(Ci::JobArtifact.metadata)).exists? + end + def has_exposed_artifacts? complete? && builds.latest.with_exposed_artifacts.exists? end @@ -925,7 +1041,7 @@ module Ci stages.find_by!(name: name) end - def error_messages + def full_error_messages errors ? errors.full_messages.to_sentence : "" end @@ -964,8 +1080,6 @@ module Ci # Set scheduling type of processables if they were created before scheduling_type # data was deployed (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22246). def ensure_scheduling_type! - return unless ::Gitlab::Ci::Features.ensure_scheduling_type_enabled? - processables.populate_scheduling_type! end @@ -977,6 +1091,12 @@ module Ci private + def add_message(severity, content) + return unless Gitlab::Ci::Features.store_pipeline_messages?(project) + + messages.build(severity: severity, content: content) + end + def pipeline_data Gitlab::DataBuilder::Pipeline.build(self) end diff --git a/app/models/ci/pipeline_enums.rb b/app/models/ci/pipeline_enums.rb index 2ccd8445aa8..352dc56aac7 100644 --- a/app/models/ci/pipeline_enums.rb +++ b/app/models/ci/pipeline_enums.rb @@ -31,7 +31,7 @@ module Ci merge_request_event: 10, external_pull_request_event: 11, parent_pipeline: 12, - ondemand_scan: 13 + ondemand_dast_scan: 13 } end @@ -45,7 +45,8 @@ module Ci webide_source: 3, remote_source: 4, external_project_source: 5, - bridge_source: 6 + bridge_source: 6, + parameter_source: 7 } end diff --git a/app/models/ci/pipeline_message.rb b/app/models/ci/pipeline_message.rb new file mode 100644 index 00000000000..a47ec554462 --- /dev/null +++ b/app/models/ci/pipeline_message.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ci + class PipelineMessage < ApplicationRecord + extend Gitlab::Ci::Model + + MAX_CONTENT_LENGTH = 10_000 + + belongs_to :pipeline + + validates :content, presence: true + + before_save :truncate_long_content + + enum severity: { error: 0, warning: 1 } + + private + + def truncate_long_content + return if content.length <= MAX_CONTENT_LENGTH + + self.content = content.truncate(MAX_CONTENT_LENGTH) + end + end +end diff --git a/app/models/ci/ref.rb b/app/models/ci/ref.rb index be6062b6e6e..29b44575d65 100644 --- a/app/models/ci/ref.rb +++ b/app/models/ci/ref.rb @@ -43,7 +43,7 @@ module Ci end def last_finished_pipeline_id - Ci::Pipeline.where(ci_ref_id: self.id).finished.order(id: :desc).select(:id).take&.id + Ci::Pipeline.last_finished_for_ref_id(self.id)&.id end def update_status_by!(pipeline) diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8fc273556f0..1cd6c64841b 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -239,6 +239,10 @@ module Ci runner_projects.count == 1 end + def belongs_to_more_than_one_project? + self.projects.limit(2).count(:all) > 1 + end + def assigned_to_group? runner_namespaces.any? end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index a316b4718e0..41215601704 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -4,10 +4,10 @@ module Ci class Stage < ApplicationRecord extend Gitlab::Ci::Model include Importable - include HasStatus + include Ci::HasStatus include Gitlab::OptimisticLocking - enum status: HasStatus::STATUSES_ENUM + enum status: Ci::HasStatus::STATUSES_ENUM belongs_to :project belongs_to :pipeline @@ -98,7 +98,7 @@ module Ci when 'scheduled' then delay when 'skipped', nil then skip else - raise HasStatus::UnknownStatusError, + raise Ci::HasStatus::UnknownStatusError, "Unknown status `#{new_status}`" end end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 08d39595c61..13358b95a47 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -18,5 +18,7 @@ module Ci } scope :unprotected, -> { where(protected: false) } + scope :by_key, -> (key) { where(key: key) } + scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) } end end diff --git a/app/models/clusters/applications/cilium.rb b/app/models/clusters/applications/cilium.rb new file mode 100644 index 00000000000..7936b0b18de --- /dev/null +++ b/app/models/clusters/applications/cilium.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Clusters + module Applications + class Cilium < ApplicationRecord + self.table_name = 'clusters_applications_cilium' + + include ::Clusters::Concerns::ApplicationCore + include ::Clusters::Concerns::ApplicationStatus + + # Cilium can only be installed and uninstalled through the + # cluster-applications project by triggering CI pipeline for a + # management project. UI operations are not available for such + # applications. More information: + # https://docs.gitlab.com/ee/user/clusters/management_project.html + def allowed_to_uninstall? + false + end + end + end +end diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 24bb1df6d22..101d782db3a 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -17,6 +17,9 @@ module Clusters default_value_for :version, VERSION + scope :preload_cluster_platform, -> { preload(cluster: [:platform_kubernetes]) } + scope :with_clusters_with_cilium, -> { joins(:cluster).merge(Clusters::Cluster.with_available_cilium) } + attr_encrypted :alert_manager_token, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 6d3b6c4ed8f..9ec7c194a26 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.17.1' + VERSION = '0.18.1' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index bde7a2104ba..7641b6d2a4b 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -2,6 +2,7 @@ module Clusters class Cluster < ApplicationRecord + prepend HasEnvironmentScope include Presentable include Gitlab::Utils::StrongMemoize include FromUnion @@ -20,7 +21,8 @@ module Clusters Clusters::Applications::Jupyter.application_name => Clusters::Applications::Jupyter, Clusters::Applications::Knative.application_name => Clusters::Applications::Knative, Clusters::Applications::ElasticStack.application_name => Clusters::Applications::ElasticStack, - Clusters::Applications::Fluentd.application_name => Clusters::Applications::Fluentd + Clusters::Applications::Fluentd.application_name => Clusters::Applications::Fluentd, + Clusters::Applications::Cilium.application_name => Clusters::Applications::Cilium }.freeze DEFAULT_ENVIRONMENT = '*' KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN' @@ -64,6 +66,7 @@ module Clusters has_one_cluster_application :knative has_one_cluster_application :elastic_stack has_one_cluster_application :fluentd + has_one_cluster_application :cilium has_many :kubernetes_namespaces has_many :metrics_dashboard_annotations, class_name: 'Metrics::Dashboard::Annotation', inverse_of: :cluster @@ -81,6 +84,7 @@ module Clusters validate :no_groups, unless: :group_type? validate :no_projects, unless: :project_type? validate :unique_management_project_environment_scope + validate :unique_environment_scope after_save :clear_reactive_cache! @@ -129,6 +133,7 @@ module Clusters scope :with_enabled_modsecurity, -> { joins(:application_ingress).merge(::Clusters::Applications::Ingress.modsecurity_enabled) } scope :with_available_elasticstack, -> { joins(:application_elastic_stack).merge(::Clusters::Applications::ElasticStack.available) } + scope :with_available_cilium, -> { joins(:application_cilium).merge(::Clusters::Applications::Cilium.available) } scope :distinct_with_deployed_environments, -> { joins(:environments).merge(::Deployment.success).distinct } scope :preload_elasticstack, -> { preload(:application_elastic_stack) } scope :preload_environments, -> { preload(:environments) } @@ -228,7 +233,9 @@ module Clusters def calculate_reactive_cache return unless enabled? - { connection_status: retrieve_connection_status, nodes: retrieve_nodes } + gitlab_kubernetes_nodes = Gitlab::Kubernetes::Node.new(self) + + { connection_status: retrieve_connection_status, nodes: gitlab_kubernetes_nodes.all.presence } end def persisted_applications @@ -335,7 +342,11 @@ module Clusters end def local_tiller_enabled? - Feature.enabled?(:managed_apps_local_tiller, clusterable, default_enabled: false) + Feature.enabled?(:managed_apps_local_tiller, clusterable, default_enabled: true) + end + + def prometheus_adapter + application_prometheus end private @@ -352,6 +363,12 @@ module Clusters end end + def unique_environment_scope + if clusterable.present? && clusterable.clusters.where(environment_scope: environment_scope).where.not(id: id).exists? + errors.add(:environment_scope, 'cannot add duplicated environment scope') + end + end + def managed_namespace(environment) Clusters::KubernetesNamespaceFinder.new( self, @@ -383,54 +400,6 @@ module Clusters result[:status] end - def retrieve_nodes - result = ::Gitlab::Kubernetes::KubeClient.graceful_request(id) { kubeclient.get_nodes } - - return unless result[:response] - - cluster_nodes = result[:response] - - result = ::Gitlab::Kubernetes::KubeClient.graceful_request(id) { kubeclient.metrics_client.get_nodes } - nodes_metrics = result[:response].to_a - - cluster_nodes.inject([]) do |memo, node| - sliced_node = filter_relevant_node_attributes(node) - - matched_node_metric = nodes_metrics.find { |node_metric| node_metric.metadata.name == node.metadata.name } - - sliced_node_metrics = matched_node_metric ? filter_relevant_node_metrics_attributes(matched_node_metric) : {} - - memo << sliced_node.merge(sliced_node_metrics) - end - end - - def filter_relevant_node_attributes(node) - { - 'metadata' => { - 'name' => node.metadata.name - }, - 'status' => { - 'capacity' => { - 'cpu' => node.status.capacity.cpu, - 'memory' => node.status.capacity.memory - }, - 'allocatable' => { - 'cpu' => node.status.allocatable.cpu, - 'memory' => node.status.allocatable.memory - } - } - } - end - - def filter_relevant_node_metrics_attributes(node_metrics) - { - 'usage' => { - 'cpu' => node_metrics.usage.cpu, - 'memory' => node_metrics.usage.memory - } - } - end - # To keep backward compatibility with AUTO_DEVOPS_DOMAIN # environment variable, we need to ensure KUBE_INGRESS_BASE_DOMAIN # is set if AUTO_DEVOPS_DOMAIN is set on any of the following options: diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 444368d0ef3..7af78960e35 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -159,7 +159,16 @@ module Clusters if ca_pem.present? opts[:cert_store] = OpenSSL::X509::Store.new - opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem)) + + file = Tempfile.new('cluster_ca_pem_temp') + begin + file.write(ca_pem) + file.rewind + opts[:cert_store].add_file(file.path) + ensure + file.close + file.unlink # deletes the temp file + end end opts diff --git a/app/models/commit.rb b/app/models/commit.rb index 681fe727456..53bcdf8165f 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -469,10 +469,12 @@ class Commit # We don't want to do anything for `Commit` model, so this is empty. end - WIP_REGEX = /\A\s*(((?i)(\[WIP\]|WIP:|WIP)\s|WIP$))|(fixup!|squash!)\s/.freeze + # WIP is deprecated in favor of Draft. Currently both options are supported + # https://gitlab.com/gitlab-org/gitlab/-/issues/227426 + DRAFT_REGEX = /\A\s*#{Regexp.union(Gitlab::Regex.merge_request_wip, Gitlab::Regex.merge_request_draft)}|(fixup!|squash!)\s/.freeze def work_in_progress? - !!(title =~ WIP_REGEX) + !!(title =~ DRAFT_REGEX) end def merged_merge_request?(user) diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb index 456d32bf403..b8653f47392 100644 --- a/app/models/commit_collection.rb +++ b/app/models/commit_collection.rb @@ -53,6 +53,17 @@ class CommitCollection self end + # Returns the collection with markdown fields preloaded. + # + # Get the markdown cache from redis using pipeline to prevent n+1 requests + # when rendering the markdown of an attribute (e.g. title, full_title, + # description). + def with_markdown_cache + Commit.preload_markdown_cache!(commits) + + self + end + def unenriched commits.reject(&:gitaly_commit?) end diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 475f82f23ca..c85292feb25 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -1,11 +1,12 @@ # frozen_string_literal: true class CommitStatus < ApplicationRecord - include HasStatus + include Ci::HasStatus include Importable include AfterCommitQueue include Presentable include EnumWithNil + include BulkInsertableAssociations self.table_name = 'ci_builds' diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb index 39e8408f794..f1c39dda49d 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage.rb @@ -125,7 +125,7 @@ module Analytics def label_available_for_group?(label_id) LabelsFinder.new(nil, { group_id: group.id, include_ancestor_groups: true, only_group_labels: true }) .execute(skip_authorization: true) - .by_ids(label_id) + .id_in(label_id) .exists? end end diff --git a/app/models/concerns/approvable_base.rb b/app/models/concerns/approvable_base.rb new file mode 100644 index 00000000000..6323bd01c58 --- /dev/null +++ b/app/models/concerns/approvable_base.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module ApprovableBase + extend ActiveSupport::Concern + + included do + has_many :approvals, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_many :approved_by_users, through: :approvals, source: :user + end + + def approved_by?(user) + return false unless user + + approved_by_users.include?(user) + end +end diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index a98baeb0e3d..ac84ef94b1c 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -36,6 +36,12 @@ module Avatarable end end + class_methods do + def bot_avatar(image:) + Rails.root.join('app', 'assets', 'images', 'bot_avatars', image).open + end + end + def avatar_type unless self.avatar.image? errors.add :avatar, "file format is not supported. Please try one of the following supported formats: #{AvatarUploader::SAFE_IMAGE_EXT.join(', ')}" diff --git a/app/models/concerns/bulk_insert_safe.rb b/app/models/concerns/bulk_insert_safe.rb index e09f44e68dc..f9eb3fb875e 100644 --- a/app/models/concerns/bulk_insert_safe.rb +++ b/app/models/concerns/bulk_insert_safe.rb @@ -37,7 +37,7 @@ module BulkInsertSafe # These are the callbacks we think safe when used on models that are # written to the database in bulk - CALLBACK_NAME_WHITELIST = Set[ + ALLOWED_CALLBACKS = Set[ :initialize, :validate, :validation, @@ -179,16 +179,12 @@ module BulkInsertSafe end def _bulk_insert_callback_allowed?(name, args) - _bulk_insert_whitelisted?(name) || _bulk_insert_saved_from_belongs_to?(name, args) + ALLOWED_CALLBACKS.include?(name) || _bulk_insert_saved_from_belongs_to?(name, args) end # belongs_to associations will install a before_save hook during class loading def _bulk_insert_saved_from_belongs_to?(name, args) args.first == :before && args.second.to_s.start_with?('autosave_associated_records_for_') end - - def _bulk_insert_whitelisted?(name) - CALLBACK_NAME_WHITELIST.include?(name) - end end end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index 7ea5382a4fa..10df5e1a8dc 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -84,8 +84,6 @@ module Ci end def secret_instance_variables - return [] unless ::Feature.enabled?(:ci_instance_level_variables, project, default_enabled: true) - project.ci_instance_variables_for(ref: git_ref) end diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb new file mode 100644 index 00000000000..c52807ec501 --- /dev/null +++ b/app/models/concerns/ci/has_status.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +module Ci + module HasStatus + extend ActiveSupport::Concern + + DEFAULT_STATUS = 'created' + BLOCKED_STATUS = %w[manual scheduled].freeze + AVAILABLE_STATUSES = %w[created waiting_for_resource preparing pending running success failed canceled skipped manual scheduled].freeze + STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze + ACTIVE_STATUSES = %w[waiting_for_resource preparing pending running].freeze + COMPLETED_STATUSES = %w[success failed canceled skipped].freeze + ORDERED_STATUSES = %w[failed preparing pending running waiting_for_resource manual scheduled canceled success skipped created].freeze + PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze + EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze + STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7, + scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze + + UnknownStatusError = Class.new(StandardError) + + class_methods do + def legacy_status_sql + scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all + scope_warnings = respond_to?(:failed_but_allowed) ? failed_but_allowed : none + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + scheduled = scope_relevant.scheduled.select('count(*)').to_sql + preparing = scope_relevant.preparing.select('count(*)').to_sql + waiting_for_resource = scope_relevant.waiting_for_resource.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql.presence || 'false' + + Arel.sql( + "(CASE + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' + WHEN (#{builds})=(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success}) THEN 'success' + WHEN (#{builds})=(#{created}) THEN 'created' + WHEN (#{builds})=(#{preparing}) THEN 'preparing' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' + WHEN (#{running})+(#{pending})>0 THEN 'running' + WHEN (#{waiting_for_resource})>0 THEN 'waiting_for_resource' + WHEN (#{manual})>0 THEN 'manual' + WHEN (#{scheduled})>0 THEN 'scheduled' + WHEN (#{preparing})>0 THEN 'preparing' + WHEN (#{created})>0 THEN 'running' + ELSE 'failed' + END)" + ) + end + + def legacy_status + all.pluck(legacy_status_sql).first + end + + # This method should not be used. + # This method performs expensive calculation of status: + # 1. By plucking all related objects, + # 2. Or executes expensive SQL query + def slow_composite_status(project:) + if ::Gitlab::Ci::Features.composite_status?(project) + Gitlab::Ci::Status::Composite + .new(all, with_allow_failure: columns_hash.key?('allow_failure')) + .status + else + legacy_status + end + end + + def started_at + all.minimum(:started_at) + end + + def finished_at + all.maximum(:finished_at) + end + + def all_state_names + state_machines.values.flat_map(&:states).flat_map { |s| s.map(&:name) } + end + + def completed_statuses + COMPLETED_STATUSES.map(&:to_sym) + end + end + + included do + validates :status, inclusion: { in: AVAILABLE_STATUSES } + + state_machine :status, initial: :created do + state :created, value: 'created' + state :waiting_for_resource, value: 'waiting_for_resource' + state :preparing, value: 'preparing' + state :pending, value: 'pending' + state :running, value: 'running' + state :failed, value: 'failed' + state :success, value: 'success' + state :canceled, value: 'canceled' + state :skipped, value: 'skipped' + state :manual, value: 'manual' + state :scheduled, value: 'scheduled' + end + + scope :created, -> { with_status(:created) } + scope :waiting_for_resource, -> { with_status(:waiting_for_resource) } + scope :preparing, -> { with_status(:preparing) } + scope :relevant, -> { without_status(:created) } + scope :running, -> { with_status(:running) } + scope :pending, -> { with_status(:pending) } + scope :success, -> { with_status(:success) } + scope :failed, -> { with_status(:failed) } + scope :canceled, -> { with_status(:canceled) } + scope :skipped, -> { with_status(:skipped) } + scope :manual, -> { with_status(:manual) } + scope :scheduled, -> { with_status(:scheduled) } + scope :alive, -> { with_status(:created, :waiting_for_resource, :preparing, :pending, :running) } + scope :alive_or_scheduled, -> { with_status(:created, :waiting_for_resource, :preparing, :pending, :running, :scheduled) } + scope :created_or_pending, -> { with_status(:created, :pending) } + scope :running_or_pending, -> { with_status(:running, :pending) } + scope :finished, -> { with_status(:success, :failed, :canceled) } + scope :failed_or_canceled, -> { with_status(:failed, :canceled) } + scope :incomplete, -> { without_statuses(completed_statuses) } + + scope :cancelable, -> do + where(status: [:running, :waiting_for_resource, :preparing, :pending, :created, :scheduled]) + end + + scope :without_statuses, -> (names) do + with_status(all_state_names - names.to_a) + end + end + + def started? + STARTED_STATUSES.include?(status) && started_at + end + + def active? + ACTIVE_STATUSES.include?(status) + end + + def complete? + COMPLETED_STATUSES.include?(status) + end + + def blocked? + BLOCKED_STATUS.include?(status) + end + + private + + def calculate_duration + if started_at && finished_at + finished_at - started_at + elsif started_at + Time.current - started_at + end + end + end +end diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index bd40af28bc9..26e644646b4 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -87,3 +87,5 @@ module Ci end end end + +Ci::Metadatable.prepend_if_ee('EE::Ci::Metadatable') diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index 3b893a56bd6..02f7711e927 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true module DeploymentPlatform - # EE would override this and utilize environment argument # rubocop:disable Gitlab/ModuleWithInstanceVariables def deployment_platform(environment: nil) @deployment_platform ||= {} @@ -20,16 +19,27 @@ module DeploymentPlatform find_instance_cluster_platform_kubernetes(environment: environment) end - # EE would override this and utilize environment argument - def find_platform_kubernetes_with_cte(_environment) - Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?).base_and_ancestors + def find_platform_kubernetes_with_cte(environment) + if environment + ::Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?) + .base_and_ancestors + .enabled + .on_environment(environment, relevant_only: true) + .first&.platform_kubernetes + else + Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?).base_and_ancestors .enabled.default_environment .first&.platform_kubernetes + end end - # EE would override this and utilize environment argument def find_instance_cluster_platform_kubernetes(environment: nil) - Clusters::Instance.new.clusters.enabled.default_environment + if environment + ::Clusters::Instance.new.clusters.enabled.on_environment(environment, relevant_only: true) .first&.platform_kubernetes + else + Clusters::Instance.new.clusters.enabled.default_environment + .first&.platform_kubernetes + end end end diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb index 29d31b8bb4f..d909b67d7ba 100644 --- a/app/models/concerns/has_repository.rb +++ b/app/models/concerns/has_repository.rb @@ -5,7 +5,7 @@ # of directly having a repository, like project or snippet. # # It also includes `Referable`, therefore the method -# `to_reference` should be overriden in case the object +# `to_reference` should be overridden in case the object # needs any special behavior. module HasRepository extend ActiveSupport::Concern @@ -76,7 +76,11 @@ module HasRepository end def default_branch - @default_branch ||= repository.root_ref + @default_branch ||= repository.root_ref || default_branch_from_preferences + end + + def default_branch_from_preferences + empty_repo? ? Gitlab::CurrentSettings.default_branch_name : nil end def reload_default_branch diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb deleted file mode 100644 index c885dea862f..00000000000 --- a/app/models/concerns/has_status.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -module HasStatus - extend ActiveSupport::Concern - - DEFAULT_STATUS = 'created' - BLOCKED_STATUS = %w[manual scheduled].freeze - AVAILABLE_STATUSES = %w[created waiting_for_resource preparing pending running success failed canceled skipped manual scheduled].freeze - STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze - ACTIVE_STATUSES = %w[waiting_for_resource preparing pending running].freeze - COMPLETED_STATUSES = %w[success failed canceled skipped].freeze - ORDERED_STATUSES = %w[failed preparing pending running waiting_for_resource manual scheduled canceled success skipped created].freeze - PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze - EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze - STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7, - scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze - - UnknownStatusError = Class.new(StandardError) - - class_methods do - def legacy_status_sql - scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all - scope_warnings = respond_to?(:failed_but_allowed) ? failed_but_allowed : none - - builds = scope_relevant.select('count(*)').to_sql - created = scope_relevant.created.select('count(*)').to_sql - success = scope_relevant.success.select('count(*)').to_sql - manual = scope_relevant.manual.select('count(*)').to_sql - scheduled = scope_relevant.scheduled.select('count(*)').to_sql - preparing = scope_relevant.preparing.select('count(*)').to_sql - waiting_for_resource = scope_relevant.waiting_for_resource.select('count(*)').to_sql - pending = scope_relevant.pending.select('count(*)').to_sql - running = scope_relevant.running.select('count(*)').to_sql - skipped = scope_relevant.skipped.select('count(*)').to_sql - canceled = scope_relevant.canceled.select('count(*)').to_sql - warnings = scope_warnings.select('count(*) > 0').to_sql.presence || 'false' - - Arel.sql( - "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' - WHEN (#{builds})=(#{skipped}) THEN 'skipped' - WHEN (#{builds})=(#{success}) THEN 'success' - WHEN (#{builds})=(#{created}) THEN 'created' - WHEN (#{builds})=(#{preparing}) THEN 'preparing' - WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' - WHEN (#{running})+(#{pending})>0 THEN 'running' - WHEN (#{waiting_for_resource})>0 THEN 'waiting_for_resource' - WHEN (#{manual})>0 THEN 'manual' - WHEN (#{scheduled})>0 THEN 'scheduled' - WHEN (#{preparing})>0 THEN 'preparing' - WHEN (#{created})>0 THEN 'running' - ELSE 'failed' - END)" - ) - end - - def legacy_status - all.pluck(legacy_status_sql).first - end - - # This method should not be used. - # This method performs expensive calculation of status: - # 1. By plucking all related objects, - # 2. Or executes expensive SQL query - def slow_composite_status(project:) - if ::Gitlab::Ci::Features.composite_status?(project) - Gitlab::Ci::Status::Composite - .new(all, with_allow_failure: columns_hash.key?('allow_failure')) - .status - else - legacy_status - end - end - - def started_at - all.minimum(:started_at) - end - - def finished_at - all.maximum(:finished_at) - end - - def all_state_names - state_machines.values.flat_map(&:states).flat_map { |s| s.map(&:name) } - end - - def completed_statuses - COMPLETED_STATUSES.map(&:to_sym) - end - end - - included do - validates :status, inclusion: { in: AVAILABLE_STATUSES } - - state_machine :status, initial: :created do - state :created, value: 'created' - state :waiting_for_resource, value: 'waiting_for_resource' - state :preparing, value: 'preparing' - state :pending, value: 'pending' - state :running, value: 'running' - state :failed, value: 'failed' - state :success, value: 'success' - state :canceled, value: 'canceled' - state :skipped, value: 'skipped' - state :manual, value: 'manual' - state :scheduled, value: 'scheduled' - end - - scope :created, -> { with_status(:created) } - scope :waiting_for_resource, -> { with_status(:waiting_for_resource) } - scope :preparing, -> { with_status(:preparing) } - scope :relevant, -> { without_status(:created) } - scope :running, -> { with_status(:running) } - scope :pending, -> { with_status(:pending) } - scope :success, -> { with_status(:success) } - scope :failed, -> { with_status(:failed) } - scope :canceled, -> { with_status(:canceled) } - scope :skipped, -> { with_status(:skipped) } - scope :manual, -> { with_status(:manual) } - scope :scheduled, -> { with_status(:scheduled) } - scope :alive, -> { with_status(:created, :waiting_for_resource, :preparing, :pending, :running) } - scope :alive_or_scheduled, -> { with_status(:created, :waiting_for_resource, :preparing, :pending, :running, :scheduled) } - scope :created_or_pending, -> { with_status(:created, :pending) } - scope :running_or_pending, -> { with_status(:running, :pending) } - scope :finished, -> { with_status(:success, :failed, :canceled) } - scope :failed_or_canceled, -> { with_status(:failed, :canceled) } - scope :incomplete, -> { without_statuses(completed_statuses) } - - scope :cancelable, -> do - where(status: [:running, :waiting_for_resource, :preparing, :pending, :created, :scheduled]) - end - - scope :without_statuses, -> (names) do - with_status(all_state_names - names.to_a) - end - end - - def started? - STARTED_STATUSES.include?(status) && started_at - end - - def active? - ACTIVE_STATUSES.include?(status) - end - - def complete? - COMPLETED_STATUSES.include?(status) - end - - def blocked? - BLOCKED_STATUS.include?(status) - end - - private - - def calculate_duration - if started_at && finished_at - finished_at - started_at - elsif started_at - Time.current - started_at - end - end -end diff --git a/app/models/concerns/integration.rb b/app/models/concerns/integration.rb index 644a0ba1b5e..34ff5bb1195 100644 --- a/app/models/concerns/integration.rb +++ b/app/models/concerns/integration.rb @@ -15,5 +15,19 @@ module Integration Project.where(id: custom_integration_project_ids) end + + def ids_without_integration(integration, limit) + services = Service + .select('1') + .where('services.project_id = projects.id') + .where(type: integration.type) + + Project + .where('NOT EXISTS (?)', services) + .where(pending_delete: false) + .where(archived: false) + .limit(limit) + .pluck(:id) + end end end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 220af8ab7c7..715cbd15d93 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -411,8 +411,8 @@ module Issuable changes = previous_changes if old_associations - old_labels = old_associations.fetch(:labels, []) - old_assignees = old_associations.fetch(:assignees, []) + old_labels = old_associations.fetch(:labels, labels) + old_assignees = old_associations.fetch(:assignees, assignees) if old_labels != labels changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)] @@ -423,7 +423,7 @@ module Issuable end if self.respond_to?(:total_time_spent) - old_total_time_spent = old_associations.fetch(:total_time_spent, nil) + old_total_time_spent = old_associations.fetch(:total_time_spent, total_time_spent) if old_total_time_spent != total_time_spent changes[:total_time_spent] = [old_total_time_spent, total_time_spent] diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index 183b902dd37..2dbe9360d42 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -67,6 +67,10 @@ module Noteable false end + def has_any_diff_note_positions? + notes.any? && DiffNotePosition.where(note: notes).exists? + end + def discussion_notes notes end diff --git a/app/models/concerns/partitioned_table.rb b/app/models/concerns/partitioned_table.rb new file mode 100644 index 00000000000..9f1cec5d520 --- /dev/null +++ b/app/models/concerns/partitioned_table.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module PartitionedTable + extend ActiveSupport::Concern + + class_methods do + attr_reader :partitioning_strategy + + PARTITIONING_STRATEGIES = { + monthly: Gitlab::Database::Partitioning::MonthlyStrategy + }.freeze + + def partitioned_by(partitioning_key, strategy:) + strategy_class = PARTITIONING_STRATEGIES[strategy.to_sym] || raise(ArgumentError, "Unknown partitioning strategy: #{strategy}") + + @partitioning_strategy = strategy_class.new(self, partitioning_key) + + Gitlab::Database::Partitioning::PartitionCreator.register(self) + end + end +end diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb index d294563139c..5f30fc0c36c 100644 --- a/app/models/concerns/reactive_caching.rb +++ b/app/models/concerns/reactive_caching.rb @@ -29,7 +29,7 @@ module ReactiveCaching self.reactive_cache_lease_timeout = 2.minutes self.reactive_cache_refresh_interval = 1.minute self.reactive_cache_lifetime = 10.minutes - self.reactive_cache_hard_limit = 1.megabyte + self.reactive_cache_hard_limit = nil # this value should be set in megabytes. E.g: 1.megabyte self.reactive_cache_work_type = :default self.reactive_cache_worker_finder = ->(id, *_args) do find_by(primary_key => id) @@ -159,8 +159,12 @@ module ReactiveCaching WORK_TYPE.fetch(self.class.reactive_cache_work_type.to_sym) end + def reactive_cache_limit_enabled? + !!self.reactive_cache_hard_limit + end + def check_exceeded_reactive_cache_limit!(data) - return unless Feature.enabled?(:reactive_cache_limit) + return unless reactive_cache_limit_enabled? data_deep_size = Gitlab::Utils::DeepSize.new(data, max_size: self.class.reactive_cache_hard_limit) diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 129d0fbb2c0..c70ce9bebcc 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -17,11 +17,8 @@ module Routable after_validation :set_path_errors - before_validation do - if full_path_changed? || full_name_changed? - prepare_route - end - end + before_validation :prepare_route + before_save :prepare_route # in case validation is skipped end class_methods do @@ -118,6 +115,8 @@ module Routable end def prepare_route + return unless full_path_changed? || full_name_changed? + route || build_route(source: self) route.path = build_full_path route.name = build_full_name diff --git a/app/models/concerns/update_project_statistics.rb b/app/models/concerns/update_project_statistics.rb index 6cf012680d8..c0fa14d3369 100644 --- a/app/models/concerns/update_project_statistics.rb +++ b/app/models/concerns/update_project_statistics.rb @@ -35,8 +35,8 @@ module UpdateProjectStatistics @project_statistics_name = project_statistics_name @statistic_attribute = statistic_attribute - after_save(:update_project_statistics_after_save, if: :update_project_statistics_attribute_changed?) - after_destroy(:update_project_statistics_after_destroy, unless: :project_destroyed?) + after_save(:update_project_statistics_after_save, if: :update_project_statistics_after_save?) + after_destroy(:update_project_statistics_after_destroy, if: :update_project_statistics_after_destroy?) end private :update_project_statistics @@ -45,6 +45,14 @@ module UpdateProjectStatistics included do private + def update_project_statistics_after_save? + update_project_statistics_attribute_changed? + end + + def update_project_statistics_after_destroy? + !project_destroyed? + end + def update_project_statistics_after_save attr = self.class.statistic_attribute delta = read_attribute(attr).to_i - attribute_before_last_save(attr).to_i diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb new file mode 100644 index 00000000000..643b4060ad6 --- /dev/null +++ b/app/models/custom_emoji.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class CustomEmoji < ApplicationRecord + belongs_to :namespace, inverse_of: :custom_emoji + + validate :valid_emoji_name + + validates :namespace, presence: true + validates :name, + uniqueness: { scope: [:namespace_id, :name] }, + presence: true, + length: { maximum: 36 }, + format: { with: /\A\w+\z/ } + + private + + def valid_emoji_name + if Gitlab::Emoji.emoji_exists?(name) + errors.add(:name, _('%{name} is already being used for another emoji') % { name: self.name }) + end + end +end diff --git a/app/models/deploy_keys_project.rb b/app/models/deploy_keys_project.rb index 40c66d5bc4c..a9cc56a7246 100644 --- a/app/models/deploy_keys_project.rb +++ b/app/models/deploy_keys_project.rb @@ -6,6 +6,7 @@ class DeployKeysProject < ApplicationRecord scope :without_project_deleted, -> { joins(:project).where(projects: { pending_delete: false }) } scope :in_project, ->(project) { where(project: project) } scope :with_write_access, -> { where(can_push: true) } + scope :with_deploy_keys, -> { includes(:deploy_key) } accepts_nested_attributes_for :deploy_key diff --git a/app/models/diff_viewer/image.rb b/app/models/diff_viewer/image.rb index cfda0058d81..62a3446a7b6 100644 --- a/app/models/diff_viewer/image.rb +++ b/app/models/diff_viewer/image.rb @@ -8,7 +8,7 @@ module DiffViewer self.partial_name = 'image' self.extensions = UploaderHelper::SAFE_IMAGE_EXT self.binary = true - self.switcher_icon = 'picture-o' + self.switcher_icon = 'doc-image' self.switcher_title = _('image diff') end end diff --git a/app/models/environment.rb b/app/models/environment.rb index 8dae2d760f5..bddc84f10b5 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -21,6 +21,7 @@ class Environment < ApplicationRecord has_many :prometheus_alerts, inverse_of: :environment has_many :metrics_dashboard_annotations, class_name: 'Metrics::Dashboard::Annotation', inverse_of: :environment has_many :self_managed_prometheus_alert_events, inverse_of: :environment + has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment has_one :last_deployment, -> { success.order('deployments.id DESC') }, class_name: 'Deployment' has_one :last_deployable, through: :last_deployment, source: 'deployable', source_type: 'CommitStatus' @@ -147,7 +148,7 @@ class Environment < ApplicationRecord Ci::Build.joins(inner_join_stop_actions) .with(cte.to_arel) .where(ci_builds[:commit_id].in(pipeline_ids)) - .where(status: HasStatus::BLOCKED_STATUS) + .where(status: Ci::HasStatus::BLOCKED_STATUS) .preload_project_and_pipeline_project .preload(:user, :metadata, :deployment) end @@ -226,6 +227,21 @@ class Environment < ApplicationRecord available? && stop_action.present? end + def cancel_deployment_jobs! + jobs = active_deployments.with_deployable + jobs.each do |deployment| + # guard against data integrity issues, + # for example https://gitlab.com/gitlab-org/gitlab/-/issues/218659#note_348823660 + next unless deployment.deployable + + Gitlab::OptimisticLocking.retry_lock(deployment.deployable) do |deployable| + deployable.cancel! if deployable&.cancelable? + end + rescue => e + Gitlab::ErrorTracking.track_exception(e, environment_id: id, deployment_id: deployment.id) + end + end + def stop_with_action!(current_user) return unless available? @@ -362,6 +378,11 @@ class Environment < ApplicationRecord def generate_slug self.slug = Gitlab::Slug::Environment.new(name).generate end + + # Overrides ReactiveCaching default to activate limit checking behind a FF + def reactive_cache_limit_enabled? + Feature.enabled?(:reactive_caching_limit_environment, project) + end end Environment.prepend_if_ee('EE::Environment') diff --git a/app/models/epic.rb b/app/models/epic.rb index e09dc1080e6..93f286f97d3 100644 --- a/app/models/epic.rb +++ b/app/models/epic.rb @@ -5,8 +5,6 @@ class Epic < ApplicationRecord include IgnorableColumns - ignore_column :health_status, remove_with: '13.0', remove_after: '2019-05-22' - def self.link_reference_pattern nil end diff --git a/app/models/event.rb b/app/models/event.rb index 9c0fcbb354b..56d7742c51a 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -83,10 +83,6 @@ class Event < ApplicationRecord scope :for_wiki_page, -> { where(target_type: 'WikiPage::Meta') } scope :for_design, -> { where(target_type: 'DesignManagement::Design') } - # Needed to implement feature flag: can be removed when feature flag is removed - scope :not_wiki_page, -> { where('target_type IS NULL or target_type <> ?', 'WikiPage::Meta') } - scope :not_design, -> { where('target_type IS NULL or target_type <> ?', 'DesignManagement::Design') } - scope :with_associations, -> do # We're using preload for "push_event_payload" as otherwise the association # is not always available (depending on the query being built). diff --git a/app/models/event_collection.rb b/app/models/event_collection.rb index 4c178e27b75..4768506b8fa 100644 --- a/app/models/event_collection.rb +++ b/app/models/event_collection.rb @@ -33,23 +33,16 @@ class EventCollection project_events end - relation = apply_feature_flags(relation) relation = paginate_events(relation) relation.with_associations.to_a end def all_project_events - apply_feature_flags(Event.from_union([project_events]).recent) + Event.from_union([project_events]).recent end private - def apply_feature_flags(events) - return events if ::Feature.enabled?(:wiki_events) - - events.not_wiki_page - end - def project_events relation_with_join_lateral('project_id', projects) end diff --git a/app/models/group.rb b/app/models/group.rb index 71f58a5fd1a..c38ddbdf6fb 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -18,6 +18,8 @@ class Group < Namespace ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 + UpdateSharedRunnersError = Class.new(StandardError) + has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members has_many :users, through: :group_members @@ -89,6 +91,8 @@ class Group < Namespace scope :with_users, -> { includes(:users) } + scope :by_id, ->(groups) { where(id: groups) } + class << self def sort_by_attribute(method) if method == 'storage_size_desc' @@ -504,6 +508,55 @@ class Group < Namespace preloader.preload(self, shared_with_group_links: [shared_with_group: :route]) end + def shared_runners_allowed? + shared_runners_enabled? || allow_descendants_override_disabled_shared_runners? + end + + def parent_allows_shared_runners? + return true unless has_parent? + + parent.shared_runners_allowed? + end + + def parent_enabled_shared_runners? + return true unless has_parent? + + parent.shared_runners_enabled? + end + + def enable_shared_runners! + raise UpdateSharedRunnersError, 'Shared Runners disabled for the parent group' unless parent_enabled_shared_runners? + + update_column(:shared_runners_enabled, true) + end + + def disable_shared_runners! + group_ids = self_and_descendants + return if group_ids.empty? + + Group.by_id(group_ids).update_all(shared_runners_enabled: false) + + all_projects.update_all(shared_runners_enabled: false) + end + + def allow_descendants_override_disabled_shared_runners! + raise UpdateSharedRunnersError, 'Shared Runners enabled' if shared_runners_enabled? + raise UpdateSharedRunnersError, 'Group level shared Runners not allowed' unless parent_allows_shared_runners? + + update_column(:allow_descendants_override_disabled_shared_runners, true) + end + + def disallow_descendants_override_disabled_shared_runners! + raise UpdateSharedRunnersError, 'Shared Runners enabled' if shared_runners_enabled? + + group_ids = self_and_descendants + return if group_ids.empty? + + Group.by_id(group_ids).update_all(allow_descendants_override_disabled_shared_runners: false) + + all_projects.update_all(shared_runners_enabled: false) + end + private def update_two_factor_requirement diff --git a/app/models/incident_management/project_incident_management_setting.rb b/app/models/incident_management/project_incident_management_setting.rb index bf57c5b883f..c79acdb685f 100644 --- a/app/models/incident_management/project_incident_management_setting.rb +++ b/app/models/incident_management/project_incident_management_setting.rb @@ -8,6 +8,15 @@ module IncidentManagement validate :issue_template_exists, if: :create_issue? + before_validation :ensure_pagerduty_token + + attr_encrypted :pagerduty_token, + mode: :per_attribute_iv, + key: ::Settings.attr_encrypted_db_key_base_truncated, + algorithm: 'aes-256-gcm', + encode: false, # No need to encode for binary column https://github.com/attr-encrypted/attr_encrypted#the-encode-encode_iv-encode_salt-and-default_encoding-options + encode_iv: false + def available_issue_templates Gitlab::Template::IssueTemplate.all(project) end @@ -30,5 +39,15 @@ module IncidentManagement Gitlab::Template::IssueTemplate.find(issue_template_key, project) rescue Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError end + + def ensure_pagerduty_token + return unless pagerduty_active + + self.pagerduty_token ||= generate_pagerduty_token + end + + def generate_pagerduty_token + SecureRandom.hex + end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 5c5190f88b1..619555f369d 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -98,6 +98,8 @@ class Issue < ApplicationRecord scope :counts_by_state, -> { reorder(nil).group(:state_id).count } + scope :service_desk, -> { where(author: ::User.support_bot) } + # An issue can be uniquely identified by project_id and iid # Takes one or more sets of composite IDs, expressed as hash-like records of # `{project_id: x, iid: y}`. @@ -373,6 +375,10 @@ class Issue < ApplicationRecord ) end + def from_service_desk? + author.id == User.support_bot.id + end + private def ensure_metrics diff --git a/app/models/issue_assignee.rb b/app/models/issue_assignee.rb index 8128b8a538e..e57acbae546 100644 --- a/app/models/issue_assignee.rb +++ b/app/models/issue_assignee.rb @@ -2,9 +2,12 @@ class IssueAssignee < ApplicationRecord belongs_to :issue - belongs_to :assignee, class_name: "User", foreign_key: :user_id + belongs_to :assignee, class_name: "User", foreign_key: :user_id, inverse_of: :issue_assignees validates :assignee, uniqueness: { scope: :issue_id } + + scope :in_projects, ->(project_ids) { joins(:issue).where("issues.project_id in (?)", project_ids) } + scope :on_issues, ->(issue_ids) { where(issue_id: issue_ids) } end IssueAssignee.prepend_if_ee('EE::IssueAssignee') diff --git a/app/models/iteration.rb b/app/models/iteration.rb index 2bda0725471..0b59cf047f7 100644 --- a/app/models/iteration.rb +++ b/app/models/iteration.rb @@ -34,6 +34,9 @@ class Iteration < ApplicationRecord .where('due_date is NULL or due_date >= ?', start_date) end + scope :start_date_passed, -> { where('start_date <= ?', Date.current).where('due_date > ?', Date.current) } + scope :due_date_passed, -> { where('due_date <= ?', Date.current) } + state_machine :state_enum, initial: :upcoming do event :start do transition upcoming: :started @@ -93,7 +96,7 @@ class Iteration < ApplicationRecord # ensure dates do not overlap with other Iterations in the same group/project def dates_do_not_overlap - return unless resource_parent.iterations.within_timeframe(start_date, due_date).exists? + return unless resource_parent.iterations.where.not(id: self.id).within_timeframe(start_date, due_date).exists? errors.add(:base, s_("Iteration|Dates cannot overlap with other existing Iterations")) end diff --git a/app/models/label.rb b/app/models/label.rb index 910cc0d68cd..3c70eef9bd5 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -149,10 +149,6 @@ class Label < ApplicationRecord 1 end - def self.by_ids(ids) - where(id: ids) - end - def self.on_project_board?(project_id, label_id) return false if label_id.blank? diff --git a/app/models/lfs_objects_project.rb b/app/models/lfs_objects_project.rb index e1966eda277..674294f0916 100644 --- a/app/models/lfs_objects_project.rb +++ b/app/models/lfs_objects_project.rb @@ -15,7 +15,7 @@ class LfsObjectsProject < ApplicationRecord enum repository_type: { project: 0, wiki: 1, - design: 2 ## EE-specific + design: 2 } scope :project_id_in, ->(ids) { where(project_id: ids) } diff --git a/app/models/member.rb b/app/models/member.rb index f2926d32d47..36f9741ce01 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -38,6 +38,11 @@ class Member < ApplicationRecord scope: [:source_type, :source_id], allow_nil: true } + validates :user_id, + uniqueness: { + message: _('project bots cannot be added to other groups / projects') + }, + if: :project_bot? # This scope encapsulates (most of) the conditions a row in the member table # must satisfy if it is a valid permission. Of particular note: @@ -473,6 +478,10 @@ class Member < ApplicationRecord def update_highest_role_attribute user_id end + + def project_bot? + user&.project_bot? + end end Member.prepend_if_ee('EE::Member') diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 9a916cd40ae..8c224dea88f 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -17,14 +17,7 @@ class GroupMember < Member scope :of_groups, ->(groups) { where(source_id: groups.select(:id)) } scope :of_ldap_type, -> { where(ldap: true) } - - scope :count_users_by_group_id, -> do - if Feature.enabled?(:optimized_count_users_by_group_id) - group(:source_id).count - else - joins(:user).group(:source_id).count - end - end + scope :count_users_by_group_id, -> { group(:source_id).count } after_create :update_two_factor_requirement, unless: :invite? after_destroy :update_two_factor_requirement, unless: :invite? diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a7e0907eb5f..b7885771781 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -20,13 +20,15 @@ class MergeRequest < ApplicationRecord include IgnorableColumns include MilestoneEventable include StateEventable + include ApprovableBase + + extend ::Gitlab::Utils::Override sha_attribute :squash_commit_sha self.reactive_cache_key = ->(model) { [model.project.id, model.iid] } self.reactive_cache_refresh_interval = 10.minutes self.reactive_cache_lifetime = 10.minutes - self.reactive_cache_hard_limit = 20.megabytes SORTING_PREFERENCE_FIELD = :merge_requests_sort @@ -103,6 +105,7 @@ class MergeRequest < ApplicationRecord after_create :ensure_merge_request_diff after_update :clear_memoized_shas + after_update :clear_memoized_source_branch_exists after_update :reload_diff_if_branch_changed after_commit :ensure_metrics, on: [:create, :update], unless: :importing? after_commit :expire_etag_cache, unless: :importing? @@ -260,6 +263,7 @@ class MergeRequest < ApplicationRecord *PROJECT_ROUTE_AND_NAMESPACE_ROUTE, metrics: [:latest_closed_by, :merged_by]) } + scope :by_target_branch_wildcard, ->(wildcard_branch_name) do where("target_branch LIKE ?", ApplicationRecord.sanitize_sql_like(wildcard_branch_name).tr('*', '%')) end @@ -386,25 +390,27 @@ class MergeRequest < ApplicationRecord end end - WIP_REGEX = /\A*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze + # WIP is deprecated in favor of Draft. Currently both options are supported + # https://gitlab.com/gitlab-org/gitlab/-/issues/227426 + DRAFT_REGEX = /\A*#{Regexp.union(Gitlab::Regex.merge_request_wip, Gitlab::Regex.merge_request_draft)}+\s*/i.freeze def self.work_in_progress?(title) - !!(title =~ WIP_REGEX) + !!(title =~ DRAFT_REGEX) end def self.wipless_title(title) - title.sub(WIP_REGEX, "") + title.sub(DRAFT_REGEX, "") end def self.wip_title(title) - work_in_progress?(title) ? title : "WIP: #{title}" + work_in_progress?(title) ? title : "Draft: #{title}" end def committers @committers ||= commits.committers end - # Verifies if title has changed not taking into account WIP prefix + # Verifies if title has changed not taking into account Draft prefix # for merge requests. def wipless_title_changed(old_title) self.class.wipless_title(old_title) != self.wipless_title @@ -858,6 +864,10 @@ class MergeRequest < ApplicationRecord clear_memoization(:target_branch_head) end + def clear_memoized_source_branch_exists + clear_memoization(:source_branch_exists) + end + def reload_diff_if_branch_changed if (saved_change_to_source_branch? || saved_change_to_target_branch?) && (source_branch_head && target_branch_head) @@ -946,7 +956,8 @@ class MergeRequest < ApplicationRecord end def can_remove_source_branch?(current_user) - !ProtectedBranch.protected?(source_project, source_branch) && + source_project && + !ProtectedBranch.protected?(source_project, source_branch) && !source_project.root_ref?(source_branch) && Ability.allowed?(current_user, :push_code, source_project) && diff_head_sha == source_branch_head.try(:sha) @@ -1017,6 +1028,10 @@ class MergeRequest < ApplicationRecord target_project != source_project end + def for_same_project? + target_project == source_project + end + # If the merge request closes any issues, save this information in the # `MergeRequestsClosingIssues` model. This is a performance optimization. # Calculating this information for a number of merge requests requires @@ -1104,9 +1119,11 @@ class MergeRequest < ApplicationRecord end def source_branch_exists? - return false unless self.source_project + strong_memoize(:source_branch_exists) do + next false unless self.source_project - self.source_project.repository.branch_exists?(self.source_branch) + self.source_project.repository.branch_exists?(self.source_branch) + end end def target_branch_exists? @@ -1142,6 +1159,13 @@ class MergeRequest < ApplicationRecord end end + def squash_on_merge? + return true if target_project.squash_always? + return false if target_project.squash_never? + + squash? + end + def has_ci? return false if has_no_commits? @@ -1273,7 +1297,7 @@ class MergeRequest < ApplicationRecord def all_pipelines strong_memoize(:all_pipelines) do - Ci::PipelinesForMergeRequestFinder.new(self).all + Ci::PipelinesForMergeRequestFinder.new(self, nil).all end end @@ -1374,9 +1398,9 @@ class MergeRequest < ApplicationRecord # TODO: consider renaming this as with exposed artifacts we generate reports, # not always compare # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224 - def compare_reports(service_class, current_user = nil) - with_reactive_cache(service_class.name, current_user&.id) do |data| - unless service_class.new(project, current_user, id: id) + def compare_reports(service_class, current_user = nil, report_type = nil ) + with_reactive_cache(service_class.name, current_user&.id, report_type) do |data| + unless service_class.new(project, current_user, id: id, report_type: report_type) .latest?(base_pipeline, actual_head_pipeline, data) raise InvalidateReactiveCache end @@ -1385,7 +1409,7 @@ class MergeRequest < ApplicationRecord end || { status: :parsing } end - def calculate_reactive_cache(identifier, current_user_id = nil, *args) + def calculate_reactive_cache(identifier, current_user_id = nil, report_type = nil, *args) service_class = identifier.constantize # TODO: the type check should change to something that includes exposed artifacts service @@ -1393,7 +1417,7 @@ class MergeRequest < ApplicationRecord raise NameError, service_class unless service_class < Ci::CompareReportsBaseService current_user = User.find_by(id: current_user_id) - service_class.new(project, current_user, id: id).execute(base_pipeline, actual_head_pipeline) + service_class.new(project, current_user, id: id, report_type: report_type).execute(base_pipeline, actual_head_pipeline) end def all_commits @@ -1582,6 +1606,23 @@ class MergeRequest < ApplicationRecord super.merge(label_url_method: :project_merge_requests_url) end + override :ensure_metrics + def ensure_metrics + MergeRequest::Metrics.safe_find_or_create_by(merge_request_id: id).tap do |metrics_record| + # Make sure we refresh the loaded association object with the newly created/loaded item. + # This is needed in order to have the exact functionality than before. + # + # Example: + # + # merge_request.metrics.destroy + # merge_request.ensure_metrics + # merge_request.metrics # should return the metrics record and not nil + # merge_request.metrics.merge_request # should return the same MR record + metrics_record.association(:merge_request).target = self + association(:metrics).target = metrics_record + end + end + private def with_rebase_lock diff --git a/app/models/merge_request_assignee.rb b/app/models/merge_request_assignee.rb index fe642bee8e2..2ac1de4321a 100644 --- a/app/models/merge_request_assignee.rb +++ b/app/models/merge_request_assignee.rb @@ -2,7 +2,9 @@ class MergeRequestAssignee < ApplicationRecord belongs_to :merge_request - belongs_to :assignee, class_name: "User", foreign_key: :user_id + belongs_to :assignee, class_name: "User", foreign_key: :user_id, inverse_of: :merge_request_assignees validates :assignee, uniqueness: { scope: :merge_request_id } + + scope :in_projects, ->(project_ids) { joins(:merge_request).where("merge_requests.target_project_id in (?)", project_ids) } end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 66b27aeac91..eb5250d5cf6 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -414,10 +414,16 @@ class MergeRequestDiff < ApplicationRecord return if stored_externally? || !use_external_diff? || merge_request_diff_files.count == 0 rows = build_merge_request_diff_files(merge_request_diff_files) + rows = build_external_merge_request_diff_files(rows) + + # Perform carrierwave activity before entering the database transaction. + # This is safe as until the `external_diff_store` column is changed, we will + # continue to consult the in-database content. + self.external_diff.store! transaction do MergeRequestDiffFile.where(merge_request_diff_id: id).delete_all - create_merge_request_diff_files(rows) + Gitlab::Database.bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert save! end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 90b4be7a674..e529ba6b486 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -13,9 +13,6 @@ class Namespace < ApplicationRecord include Gitlab::Utils::StrongMemoize include IgnorableColumns - ignore_column :plan_id, remove_with: '13.1', remove_after: '2020-06-22' - ignore_column :trial_ends_on, remove_with: '13.2', remove_after: '2020-07-22' - # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of # Android repo (15) + some extra backup. @@ -25,6 +22,7 @@ class Namespace < ApplicationRecord has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics + has_one :namespace_settings, inverse_of: :namespace, class_name: 'NamespaceSetting', autosave: true has_many :runner_namespaces, inverse_of: :namespace, class_name: 'Ci::RunnerNamespace' has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' @@ -35,6 +33,7 @@ class Namespace < ApplicationRecord belongs_to :parent, class_name: "Namespace" has_many :children, class_name: "Namespace", foreign_key: :parent_id + has_many :custom_emoji, inverse_of: :namespace has_one :chat_team, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :root_storage_statistics, class_name: 'Namespace::RootStorageStatistics' has_one :aggregation_schedule, class_name: 'Namespace::AggregationSchedule' @@ -50,6 +49,13 @@ class Namespace < ApplicationRecord length: { maximum: 255 }, namespace_path: true + # Introduce minimal path length of 2 characters. + # Allow change of other attributes without forcing users to + # rename their user or group. At the same time prevent changing + # the path without complying with new 2 chars requirement. + # Issue https://gitlab.com/gitlab-org/gitlab/-/issues/225214 + validates :path, length: { minimum: 2 }, if: :path_changed? + validates :max_artifacts_size, numericality: { only_integer: true, greater_than: 0, allow_nil: true } validate :nesting_level_allowed @@ -82,6 +88,7 @@ class Namespace < ApplicationRecord 'COALESCE(SUM(ps.storage_size), 0) AS storage_size', 'COALESCE(SUM(ps.repository_size), 0) AS repository_size', 'COALESCE(SUM(ps.wiki_size), 0) AS wiki_size', + 'COALESCE(SUM(ps.snippets_size), 0) AS snippets_size', 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', 'COALESCE(SUM(ps.packages_size), 0) AS packages_size' @@ -212,7 +219,7 @@ class Namespace < ApplicationRecord Gitlab.config.lfs.enabled end - def shared_runners_enabled? + def any_project_with_shared_runners_enabled? projects.with_shared_runners.any? end @@ -281,6 +288,8 @@ class Namespace < ApplicationRecord end def root_ancestor + return self if persisted? && parent_id.nil? + strong_memoize(:root_ancestor) do self_and_ancestors.reorder(nil).find_by(parent_id: nil) end diff --git a/app/models/namespace/root_storage_size.rb b/app/models/namespace/root_storage_size.rb deleted file mode 100644 index d61917e468e..00000000000 --- a/app/models/namespace/root_storage_size.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -class Namespace::RootStorageSize - def initialize(root_namespace) - @root_namespace = root_namespace - end - - def above_size_limit? - return false if limit == 0 - - usage_ratio > 1 - end - - def usage_ratio - return 0 if limit == 0 - - current_size.to_f / limit.to_f - end - - def current_size - @current_size ||= root_namespace.root_storage_statistics&.storage_size - end - - def limit - @limit ||= Gitlab::CurrentSettings.namespace_storage_size_limit.megabytes - end - - private - - attr_reader :root_namespace -end diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index ae9b2f14343..2ad6ea59588 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true class Namespace::RootStorageStatistics < ApplicationRecord - STATISTICS_ATTRIBUTES = %w(storage_size repository_size wiki_size lfs_objects_size build_artifacts_size packages_size).freeze + SNIPPETS_SIZE_STAT_NAME = 'snippets_size'.freeze + STATISTICS_ATTRIBUTES = %W(storage_size repository_size wiki_size lfs_objects_size build_artifacts_size packages_size #{SNIPPETS_SIZE_STAT_NAME}).freeze self.primary_key = :namespace_id @@ -13,11 +14,15 @@ class Namespace::RootStorageStatistics < ApplicationRecord delegate :all_projects, to: :namespace def recalculate! - update!(attributes_from_project_statistics) + update!(merged_attributes) end private + def merged_attributes + attributes_from_project_statistics.merge!(attributes_from_personal_snippets) { |key, v1, v2| v1 + v2 } + end + def attributes_from_project_statistics from_project_statistics .take @@ -34,7 +39,22 @@ class Namespace::RootStorageStatistics < ApplicationRecord 'COALESCE(SUM(ps.wiki_size), 0) AS wiki_size', 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', - 'COALESCE(SUM(ps.packages_size), 0) AS packages_size' + 'COALESCE(SUM(ps.packages_size), 0) AS packages_size', + "COALESCE(SUM(ps.snippets_size), 0) AS #{SNIPPETS_SIZE_STAT_NAME}" ) end + + def attributes_from_personal_snippets + # Return if the type of namespace does not belong to a user + return {} unless namespace.type.nil? + + from_personal_snippets.take.slice(SNIPPETS_SIZE_STAT_NAME) + end + + def from_personal_snippets + PersonalSnippet + .joins('INNER JOIN snippet_statistics s ON s.snippet_id = snippets.id') + .where(author: namespace.owner_id) + .select("COALESCE(SUM(s.repository_size), 0) AS #{SNIPPETS_SIZE_STAT_NAME}") + end end diff --git a/app/models/namespace/traversal_hierarchy.rb b/app/models/namespace/traversal_hierarchy.rb new file mode 100644 index 00000000000..cfb6cfdde74 --- /dev/null +++ b/app/models/namespace/traversal_hierarchy.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true +# +# A Namespace::TraversalHierarchy is the collection of namespaces that descend +# from a root Namespace as defined by the Namespace#traversal_ids attributes. +# +# This class provides operations to be performed on the hierarchy itself, +# rather than individual namespaces. +# +# This includes methods for synchronizing traversal_ids attributes to a correct +# state. We use recursive methods to determine the correct state so we don't +# have to depend on the integrity of the traversal_ids attribute values +# themselves. +# +class Namespace + class TraversalHierarchy + attr_accessor :root + + def self.for_namespace(namespace) + new(recursive_root_ancestor(namespace)) + end + + def initialize(root) + raise StandardError.new('Must specify a root node') if root.parent_id + + @root = root + end + + # Update all traversal_ids in the current namespace hierarchy. + def sync_traversal_ids! + # An issue in Rails since 2013 prevents this kind of join based update in + # ActiveRecord. https://github.com/rails/rails/issues/13496 + # Ideally it would be: + # `incorrect_traversal_ids.update_all('traversal_ids = cte.traversal_ids')` + sql = """ + UPDATE namespaces + SET traversal_ids = cte.traversal_ids + FROM (#{recursive_traversal_ids}) as cte + WHERE namespaces.id = cte.id + AND namespaces.traversal_ids <> cte.traversal_ids + """ + Namespace.connection.exec_query(sql) + end + + # Identify all incorrect traversal_ids in the current namespace hierarchy. + def incorrect_traversal_ids + Namespace + .joins("INNER JOIN (#{recursive_traversal_ids}) as cte ON namespaces.id = cte.id") + .where('namespaces.traversal_ids <> cte.traversal_ids') + end + + private + + # Determine traversal_ids for the namespace hierarchy using recursive methods. + # Generate a collection of [id, traversal_ids] rows. + # + # Note that the traversal_ids represent a calculated traversal path for the + # namespace and not the value stored within the traversal_ids attribute. + def recursive_traversal_ids + root_id = Integer(@root.id) + + """ + WITH RECURSIVE cte(id, traversal_ids, cycle) AS ( + VALUES(#{root_id}, ARRAY[#{root_id}], false) + UNION ALL + SELECT n.id, cte.traversal_ids || n.id, n.id = ANY(cte.traversal_ids) + FROM namespaces n, cte + WHERE n.parent_id = cte.id AND NOT cycle + ) + SELECT id, traversal_ids FROM cte + """ + end + + # This is essentially Namespace#root_ancestor which will soon be rewritten + # to use traversal_ids. We replicate here as a reliable way to find the + # root using recursive methods. + def self.recursive_root_ancestor(namespace) + Gitlab::ObjectHierarchy + .new(Namespace.where(id: namespace)) + .base_and_ancestors + .reorder(nil) + .find_by(parent_id: nil) + end + end +end diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb new file mode 100644 index 00000000000..53bfa3d979e --- /dev/null +++ b/app/models/namespace_setting.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class NamespaceSetting < ApplicationRecord + belongs_to :namespace, inverse_of: :namespace_settings + + self.primary_key = :namespace_id +end + +NamespaceSetting.prepend_if_ee('EE::NamespaceSetting') diff --git a/app/models/note.rb b/app/models/note.rb index 6b6a7c50b00..2db7e4e406d 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -5,6 +5,7 @@ # A note of this type is never resolvable. class Note < ApplicationRecord extend ActiveModel::Naming + include Gitlab::Utils::StrongMemoize include Participable include Mentionable include Awardable @@ -122,6 +123,8 @@ class Note < ApplicationRecord scope :common, -> { where(noteable_type: ["", nil]) } scope :fresh, -> { order(created_at: :asc, id: :asc) } scope :updated_after, ->(time) { where('updated_at > ?', time) } + scope :with_updated_at, ->(time) { where(updated_at: time) } + scope :by_updated_at, -> { reorder(:updated_at, :id) } scope :inc_author_project, -> { includes(:project, :author) } scope :inc_author, -> { includes(:author) } scope :inc_relations_for_view, -> do @@ -446,8 +449,10 @@ class Note < ApplicationRecord # Consider using `#to_discussion` if we do not need to render the discussion # and all its notes and if we don't care about the discussion's resolvability status. def discussion - full_discussion = self.noteable.notes.find_discussion(self.discussion_id) if part_of_discussion? - full_discussion || to_discussion + strong_memoize(:discussion) do + full_discussion = self.noteable.notes.find_discussion(self.discussion_id) if part_of_discussion? + full_discussion || to_discussion + end end def start_of_discussion? diff --git a/app/models/packages.rb b/app/models/packages.rb new file mode 100644 index 00000000000..e14c9290093 --- /dev/null +++ b/app/models/packages.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module Packages + def self.table_name_prefix + 'packages_' + end +end diff --git a/app/models/packages/build_info.rb b/app/models/packages/build_info.rb new file mode 100644 index 00000000000..df8cf68490e --- /dev/null +++ b/app/models/packages/build_info.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class Packages::BuildInfo < ApplicationRecord + belongs_to :package, inverse_of: :build_info + belongs_to :pipeline, class_name: 'Ci::Pipeline' +end diff --git a/app/models/packages/composer/metadatum.rb b/app/models/packages/composer/metadatum.rb new file mode 100644 index 00000000000..3026f5ea878 --- /dev/null +++ b/app/models/packages/composer/metadatum.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Packages + module Composer + class Metadatum < ApplicationRecord + self.table_name = 'packages_composer_metadata' + self.primary_key = :package_id + + belongs_to :package, -> { where(package_type: :composer) }, inverse_of: :composer_metadatum + + validates :package, :target_sha, :composer_json, presence: true + end + end +end diff --git a/app/models/packages/conan.rb b/app/models/packages/conan.rb new file mode 100644 index 00000000000..01007c3fa78 --- /dev/null +++ b/app/models/packages/conan.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true +module Packages + module Conan + def self.table_name_prefix + 'packages_conan_' + end + end +end diff --git a/app/models/packages/conan/file_metadatum.rb b/app/models/packages/conan/file_metadatum.rb new file mode 100644 index 00000000000..e1ef62b3959 --- /dev/null +++ b/app/models/packages/conan/file_metadatum.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class Packages::Conan::FileMetadatum < ApplicationRecord + belongs_to :package_file, inverse_of: :conan_file_metadatum + + validates :package_file, presence: true + + validates :recipe_revision, + presence: true, + format: { with: Gitlab::Regex.conan_revision_regex } + + validates :package_revision, absence: true, if: :recipe_file? + validates :package_revision, format: { with: Gitlab::Regex.conan_revision_regex }, if: :package_file? + + validates :conan_package_reference, absence: true, if: :recipe_file? + validates :conan_package_reference, format: { with: Gitlab::Regex.conan_package_reference_regex }, if: :package_file? + validate :conan_package_type + + enum conan_file_type: { recipe_file: 1, package_file: 2 } + + RECIPE_FILES = ::Gitlab::Regex::Packages::CONAN_RECIPE_FILES + PACKAGE_FILES = ::Gitlab::Regex::Packages::CONAN_PACKAGE_FILES + PACKAGE_BINARY = 'conan_package.tgz' + + private + + def conan_package_type + unless package_file&.package&.conan? + errors.add(:base, _('Package type must be Conan')) + end + end +end diff --git a/app/models/packages/conan/metadatum.rb b/app/models/packages/conan/metadatum.rb new file mode 100644 index 00000000000..7ec2641177a --- /dev/null +++ b/app/models/packages/conan/metadatum.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Packages::Conan::Metadatum < ApplicationRecord + belongs_to :package, -> { where(package_type: :conan) }, inverse_of: :conan_metadatum + + validates :package, presence: true + + validates :package_username, + presence: true, + format: { with: Gitlab::Regex.conan_recipe_component_regex } + + validates :package_channel, + presence: true, + format: { with: Gitlab::Regex.conan_recipe_component_regex } + + validate :conan_package_type + + def recipe + "#{package.name}/#{package.version}@#{package_username}/#{package_channel}" + end + + def recipe_path + recipe.tr('@', '/') + end + + def self.package_username_from(full_path:) + full_path.tr('/', '+') + end + + def self.full_path_from(package_username:) + package_username.tr('+', '/') + end + + private + + def conan_package_type + unless package&.conan? + errors.add(:base, _('Package type must be Conan')) + end + end +end diff --git a/app/models/packages/dependency.rb b/app/models/packages/dependency.rb new file mode 100644 index 00000000000..51b80934827 --- /dev/null +++ b/app/models/packages/dependency.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +class Packages::Dependency < ApplicationRecord + has_many :dependency_links, class_name: 'Packages::DependencyLink' + + validates :name, :version_pattern, presence: true + + validates :name, uniqueness: { scope: :version_pattern } + + NAME_VERSION_PATTERN_TUPLE_MATCHING = '(name, version_pattern) = (?, ?)'.freeze + MAX_STRING_LENGTH = 255.freeze + MAX_CHUNKED_QUERIES_COUNT = 10.freeze + + def self.ids_for_package_names_and_version_patterns(names_and_version_patterns = {}, chunk_size = 50, max_rows_limit = 200) + names_and_version_patterns.reject! { |key, value| key.size > MAX_STRING_LENGTH || value.size > MAX_STRING_LENGTH } + raise ArgumentError, 'Too many names_and_version_patterns' if names_and_version_patterns.size > MAX_CHUNKED_QUERIES_COUNT * chunk_size + + matched_ids = [] + names_and_version_patterns.each_slice(chunk_size) do |tuples| + where_statement = Array.new(tuples.size, NAME_VERSION_PATTERN_TUPLE_MATCHING) + .join(' OR ') + ids = where(where_statement, *tuples.flatten) + .limit(max_rows_limit + 1) + .pluck(:id) + matched_ids.concat(ids) + + raise ArgumentError, 'Too many Dependencies selected' if matched_ids.size > max_rows_limit + end + + matched_ids + end + + def self.for_package_names_and_version_patterns(names_and_version_patterns = {}, chunk_size = 50, max_rows_limit = 200) + ids = ids_for_package_names_and_version_patterns(names_and_version_patterns, chunk_size, max_rows_limit) + + return none if ids.empty? + + id_in(ids) + end + + def self.pluck_ids_and_names + pluck(:id, :name) + end + + def orphaned? + self.dependency_links.empty? + end +end diff --git a/app/models/packages/dependency_link.rb b/app/models/packages/dependency_link.rb new file mode 100644 index 00000000000..51018602bdc --- /dev/null +++ b/app/models/packages/dependency_link.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +class Packages::DependencyLink < ApplicationRecord + belongs_to :package, inverse_of: :dependency_links + belongs_to :dependency, inverse_of: :dependency_links, class_name: 'Packages::Dependency' + has_one :nuget_metadatum, inverse_of: :dependency_link, class_name: 'Packages::Nuget::DependencyLinkMetadatum' + + validates :package, :dependency, presence: true + + validates :dependency_type, + uniqueness: { scope: %i[package_id dependency_id] } + + enum dependency_type: { dependencies: 1, devDependencies: 2, bundleDependencies: 3, peerDependencies: 4 } + + scope :with_dependency_type, ->(dependency_type) { where(dependency_type: dependency_type) } + scope :includes_dependency, -> { includes(:dependency) } + scope :for_package, ->(package) { where(package_id: package.id) } + scope :preload_dependency, -> { preload(:dependency) } + scope :preload_nuget_metadatum, -> { preload(:nuget_metadatum) } +end diff --git a/app/models/packages/go/module.rb b/app/models/packages/go/module.rb new file mode 100644 index 00000000000..b38b691ed6c --- /dev/null +++ b/app/models/packages/go/module.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Packages + module Go + class Module + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :name, :path + + def initialize(project, name, path) + @project = project + @name = name + @path = path + end + + def versions + strong_memoize(:versions) { Packages::Go::VersionFinder.new(self).execute } + end + + def version_by(ref: nil, commit: nil) + raise ArgumentError.new 'no filter specified' unless ref || commit + raise ArgumentError.new 'ref and commit are mutually exclusive' if ref && commit + + if commit + return version_by_sha(commit) if commit.is_a? String + + return version_by_commit(commit) + end + + return version_by_name(ref) if ref.is_a? String + + version_by_ref(ref) + end + + def path_valid?(major) + m = /\/v(\d+)$/i.match(@name) + + case major + when 0, 1 + m.nil? + else + !m.nil? && m[1].to_i == major + end + end + + def gomod_valid?(gomod) + if Feature.enabled?(:go_proxy_disable_gomod_validation, @project) + return gomod&.start_with?("module ") + end + + gomod&.split("\n", 2)&.first == "module #{@name}" + end + + private + + def version_by_name(name) + # avoid a Gitaly call if possible + if strong_memoized?(:versions) + v = versions.find { |v| v.name == ref } + return v if v + end + + ref = @project.repository.find_tag(name) || @project.repository.find_branch(name) + return unless ref + + version_by_ref(ref) + end + + def version_by_ref(ref) + # reuse existing versions + if strong_memoized?(:versions) + v = versions.find { |v| v.ref == ref } + return v if v + end + + commit = ref.dereferenced_target + semver = Packages::SemVer.parse(ref.name, prefixed: true) + Packages::Go::ModuleVersion.new(self, :ref, commit, ref: ref, semver: semver) + end + + def version_by_sha(sha) + commit = @project.commit_by(oid: sha) + return unless ref + + version_by_commit(commit) + end + + def version_by_commit(commit) + Packages::Go::ModuleVersion.new(self, :commit, commit) + end + end + end +end diff --git a/app/models/packages/go/module_version.rb b/app/models/packages/go/module_version.rb new file mode 100644 index 00000000000..a50c78f8e69 --- /dev/null +++ b/app/models/packages/go/module_version.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module Packages + module Go + class ModuleVersion + include Gitlab::Utils::StrongMemoize + + VALID_TYPES = %i[ref commit pseudo].freeze + + attr_reader :mod, :type, :ref, :commit + + delegate :major, to: :@semver, allow_nil: true + delegate :minor, to: :@semver, allow_nil: true + delegate :patch, to: :@semver, allow_nil: true + delegate :prerelease, to: :@semver, allow_nil: true + delegate :build, to: :@semver, allow_nil: true + + def initialize(mod, type, commit, name: nil, semver: nil, ref: nil) + raise ArgumentError.new("invalid type '#{type}'") unless VALID_TYPES.include? type + raise ArgumentError.new("mod is required") unless mod + raise ArgumentError.new("commit is required") unless commit + + if type == :ref + raise ArgumentError.new("ref is required") unless ref + elsif type == :pseudo + raise ArgumentError.new("name is required") unless name + raise ArgumentError.new("semver is required") unless semver + end + + @mod = mod + @type = type + @commit = commit + @name = name if name + @semver = semver if semver + @ref = ref if ref + end + + def name + @name || @ref&.name + end + + def full_name + "#{mod.name}@#{name || commit.sha}" + end + + def gomod + strong_memoize(:gomod) do + if strong_memoized?(:blobs) + blob_at(@mod.path + '/go.mod') + elsif @mod.path.empty? + @mod.project.repository.blob_at(@commit.sha, 'go.mod')&.data + else + @mod.project.repository.blob_at(@commit.sha, @mod.path + '/go.mod')&.data + end + end + end + + def archive + suffix_len = @mod.path == '' ? 0 : @mod.path.length + 1 + + Zip::OutputStream.write_buffer do |zip| + files.each do |file| + zip.put_next_entry "#{full_name}/#{file[suffix_len...]}" + zip.write blob_at(file) + end + end + end + + def files + strong_memoize(:files) do + ls_tree.filter { |e| !excluded.any? { |n| e.start_with? n } } + end + end + + def excluded + strong_memoize(:excluded) do + ls_tree + .filter { |f| f.end_with?('/go.mod') && f != @mod.path + '/go.mod' } + .map { |f| f[0..-7] } + end + end + + def valid? + @mod.path_valid?(major) && @mod.gomod_valid?(gomod) + end + + private + + def blob_at(path) + return if path.nil? || path.empty? + + path = path[1..] if path.start_with? '/' + + blobs.find { |x| x.path == path }&.data + end + + def blobs + strong_memoize(:blobs) { @mod.project.repository.batch_blobs(files.map { |x| [@commit.sha, x] }) } + end + + def ls_tree + strong_memoize(:ls_tree) do + path = + if @mod.path.empty? + '.' + else + @mod.path + end + + @mod.project.repository.gitaly_repository_client.search_files_by_name(@commit.sha, path) + end + end + end + end +end diff --git a/app/models/packages/maven.rb b/app/models/packages/maven.rb new file mode 100644 index 00000000000..5c1581ce0b7 --- /dev/null +++ b/app/models/packages/maven.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true +module Packages + module Maven + def self.table_name_prefix + 'packages_maven_' + end + end +end diff --git a/app/models/packages/maven/metadatum.rb b/app/models/packages/maven/metadatum.rb new file mode 100644 index 00000000000..b7f27fb9e06 --- /dev/null +++ b/app/models/packages/maven/metadatum.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +class Packages::Maven::Metadatum < ApplicationRecord + belongs_to :package, -> { where(package_type: :maven) } + + validates :package, presence: true + + validates :path, + presence: true, + format: { with: Gitlab::Regex.maven_path_regex } + + validates :app_group, + presence: true, + format: { with: Gitlab::Regex.maven_app_group_regex } + + validates :app_name, + presence: true, + format: { with: Gitlab::Regex.maven_app_name_regex } + + validate :maven_package_type + + private + + def maven_package_type + unless package&.maven? + errors.add(:base, _('Package type must be Maven')) + end + end +end diff --git a/app/models/packages/nuget.rb b/app/models/packages/nuget.rb new file mode 100644 index 00000000000..42c167e9b7f --- /dev/null +++ b/app/models/packages/nuget.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true +module Packages + module Nuget + def self.table_name_prefix + 'packages_nuget_' + end + end +end diff --git a/app/models/packages/nuget/dependency_link_metadatum.rb b/app/models/packages/nuget/dependency_link_metadatum.rb new file mode 100644 index 00000000000..b586b55d3f0 --- /dev/null +++ b/app/models/packages/nuget/dependency_link_metadatum.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Packages::Nuget::DependencyLinkMetadatum < ApplicationRecord + self.primary_key = :dependency_link_id + + belongs_to :dependency_link, inverse_of: :nuget_metadatum + + validates :dependency_link, :target_framework, presence: true + + validate :ensure_nuget_package_type + + private + + def ensure_nuget_package_type + return if dependency_link&.package&.nuget? + + errors.add(:base, _('Package type must be NuGet')) + end +end diff --git a/app/models/packages/nuget/metadatum.rb b/app/models/packages/nuget/metadatum.rb new file mode 100644 index 00000000000..1db8c0eddbf --- /dev/null +++ b/app/models/packages/nuget/metadatum.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class Packages::Nuget::Metadatum < ApplicationRecord + belongs_to :package, -> { where(package_type: :nuget) }, inverse_of: :nuget_metadatum + + validates :package, presence: true + validates :license_url, public_url: { allow_blank: true } + validates :project_url, public_url: { allow_blank: true } + validates :icon_url, public_url: { allow_blank: true } + + validate :ensure_at_least_one_field_supplied + validate :ensure_nuget_package_type + + private + + def ensure_at_least_one_field_supplied + return if license_url? || project_url? || icon_url? + + errors.add(:base, _('Nuget metadatum must have at least license_url, project_url or icon_url set')) + end + + def ensure_nuget_package_type + return if package&.nuget? + + errors.add(:base, _('Package type must be NuGet')) + end +end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb new file mode 100644 index 00000000000..d6633456de4 --- /dev/null +++ b/app/models/packages/package.rb @@ -0,0 +1,195 @@ +# frozen_string_literal: true +class Packages::Package < ApplicationRecord + include Sortable + include Gitlab::SQL::Pattern + include UsageStatistics + + belongs_to :project + # package_files must be destroyed by ruby code in order to properly remove carrierwave uploads and update project statistics + has_many :package_files, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :dependency_links, inverse_of: :package, class_name: 'Packages::DependencyLink' + has_many :tags, inverse_of: :package, class_name: 'Packages::Tag' + has_one :conan_metadatum, inverse_of: :package, class_name: 'Packages::Conan::Metadatum' + has_one :pypi_metadatum, inverse_of: :package, class_name: 'Packages::Pypi::Metadatum' + has_one :maven_metadatum, inverse_of: :package, class_name: 'Packages::Maven::Metadatum' + has_one :nuget_metadatum, inverse_of: :package, class_name: 'Packages::Nuget::Metadatum' + has_one :composer_metadatum, inverse_of: :package, class_name: 'Packages::Composer::Metadatum' + has_one :build_info, inverse_of: :package + + accepts_nested_attributes_for :conan_metadatum + accepts_nested_attributes_for :maven_metadatum + + delegate :recipe, :recipe_path, to: :conan_metadatum, prefix: :conan + + validates :project, presence: true + validates :name, presence: true + + validates :name, format: { with: Gitlab::Regex.package_name_regex }, unless: :conan? + + validates :name, + uniqueness: { scope: %i[project_id version package_type] }, unless: :conan? + + validate :valid_conan_package_recipe, if: :conan? + validate :valid_npm_package_name, if: :npm? + validate :valid_composer_global_name, if: :composer? + validate :package_already_taken, if: :npm? + validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { npm? || nuget? } + validates :name, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? + validates :version, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? + validates :version, format: { with: Gitlab::Regex.maven_version_regex }, if: -> { version? && maven? } + + enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5, composer: 6 } + + scope :with_name, ->(name) { where(name: name) } + scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) } + scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } + scope :with_version, ->(version) { where(version: version) } + scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } + scope :with_package_type, ->(package_type) { where(package_type: package_type) } + + scope :with_conan_channel, ->(package_channel) do + joins(:conan_metadatum).where(packages_conan_metadata: { package_channel: package_channel }) + end + scope :with_conan_username, ->(package_username) do + joins(:conan_metadatum).where(packages_conan_metadata: { package_username: package_username }) + end + + scope :with_composer_target, -> (target) do + includes(:composer_metadatum) + .joins(:composer_metadatum) + .where(Packages::Composer::Metadatum.table_name => { target_sha: target }) + end + scope :preload_composer, -> { preload(:composer_metadatum) } + + scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) } + + scope :has_version, -> { where.not(version: nil) } + scope :processed, -> do + where.not(package_type: :nuget).or( + where.not(name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) + ) + end + scope :preload_files, -> { preload(:package_files) } + scope :last_of_each_version, -> { where(id: all.select('MAX(id) AS id').group(:version)) } + scope :limit_recent, ->(limit) { order_created_desc.limit(limit) } + scope :select_distinct_name, -> { select(:name).distinct } + + # Sorting + scope :order_created, -> { reorder('created_at ASC') } + scope :order_created_desc, -> { reorder('created_at DESC') } + scope :order_name, -> { reorder('name ASC') } + scope :order_name_desc, -> { reorder('name DESC') } + scope :order_version, -> { reorder('version ASC') } + scope :order_version_desc, -> { reorder('version DESC') } + scope :order_type, -> { reorder('package_type ASC') } + scope :order_type_desc, -> { reorder('package_type DESC') } + scope :order_project_name, -> { joins(:project).reorder('projects.name ASC') } + scope :order_project_name_desc, -> { joins(:project).reorder('projects.name DESC') } + scope :order_project_path, -> { joins(:project).reorder('projects.path ASC, id ASC') } + scope :order_project_path_desc, -> { joins(:project).reorder('projects.path DESC, id DESC') } + + def self.for_projects(projects) + return none unless projects.any? + + where(project_id: projects) + end + + def self.only_maven_packages_with_path(path) + joins(:maven_metadatum).where(packages_maven_metadata: { path: path }) + end + + def self.by_name_and_file_name(name, file_name) + with_name(name) + .joins(:package_files) + .where(packages_package_files: { file_name: file_name }).last! + end + + def self.by_file_name_and_sha256(file_name, sha256) + joins(:package_files) + .where(packages_package_files: { file_name: file_name, file_sha256: sha256 }).last! + end + + def self.pluck_names + pluck(:name) + end + + def self.pluck_versions + pluck(:version) + end + + def self.sort_by_attribute(method) + case method.to_s + when 'created_asc' then order_created + when 'created_at_asc' then order_created + when 'name_asc' then order_name + when 'name_desc' then order_name_desc + when 'version_asc' then order_version + when 'version_desc' then order_version_desc + when 'type_asc' then order_type + when 'type_desc' then order_type_desc + when 'project_name_asc' then order_project_name + when 'project_name_desc' then order_project_name_desc + when 'project_path_asc' then order_project_path + when 'project_path_desc' then order_project_path_desc + else + order_created_desc + end + end + + def versions + project.packages + .with_name(name) + .where.not(version: version) + .with_package_type(package_type) + .order(:version) + end + + def pipeline + build_info&.pipeline + end + + def tag_names + tags.pluck(:name) + end + + private + + def valid_conan_package_recipe + recipe_exists = project.packages + .conan + .includes(:conan_metadatum) + .with_name(name) + .with_version(version) + .with_conan_channel(conan_metadatum.package_channel) + .with_conan_username(conan_metadatum.package_username) + .id_not_in(id) + .exists? + + errors.add(:base, _('Package recipe already exists')) if recipe_exists + end + + def valid_composer_global_name + # .default_scoped is required here due to a bug in rails that leaks + # the scope and adds `self` to the query incorrectly + # See https://github.com/rails/rails/pull/35186 + if Packages::Package.default_scoped.composer.with_name(name).where.not(project_id: project_id).exists? + errors.add(:name, 'is already taken by another project') + end + end + + def valid_npm_package_name + return unless project&.root_namespace + + unless name =~ %r{\A@#{project.root_namespace.path}/[^/]+\z} + errors.add(:name, 'is not valid') + end + end + + def package_already_taken + return unless project + + if project.package_already_taken?(name) + errors.add(:base, _('Package already exists')) + end + end +end diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb new file mode 100644 index 00000000000..567b5a14603 --- /dev/null +++ b/app/models/packages/package_file.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true +class Packages::PackageFile < ApplicationRecord + include UpdateProjectStatistics + + delegate :project, :project_id, to: :package + delegate :conan_file_type, to: :conan_file_metadatum + + belongs_to :package + + has_one :conan_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Conan::FileMetadatum' + + accepts_nested_attributes_for :conan_file_metadatum + + validates :package, presence: true + validates :file, presence: true + validates :file_name, presence: true + + scope :recent, -> { order(id: :desc) } + scope :with_file_name, ->(file_name) { where(file_name: file_name) } + scope :with_file_name_like, ->(file_name) { where(arel_table[:file_name].matches(file_name)) } + scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) } + scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) } + + scope :with_conan_file_type, ->(file_type) do + joins(:conan_file_metadatum) + .where(packages_conan_file_metadata: { conan_file_type: ::Packages::Conan::FileMetadatum.conan_file_types[file_type] }) + end + + scope :with_conan_package_reference, ->(conan_package_reference) do + joins(:conan_file_metadatum) + .where(packages_conan_file_metadata: { conan_package_reference: conan_package_reference }) + end + + mount_uploader :file, Packages::PackageFileUploader + + after_save :update_file_metadata, if: :saved_change_to_file? + + update_project_statistics project_statistics_name: :packages_size + + def update_file_metadata + # The file.object_store is set during `uploader.store!` + # which happens after object is inserted/updated + self.update_column(:file_store, file.object_store) + self.update_column(:size, file.size) unless file.size == self.size + end + + def download_path + Gitlab::Routing.url_helpers.download_project_package_file_path(project, self) if ::Gitlab.ee? + end + + def local? + file_store == ::Packages::PackageFileUploader::Store::LOCAL + end +end + +Packages::PackageFile.prepend_if_ee('EE::Packages::PackageFileGeo') diff --git a/app/models/packages/pypi.rb b/app/models/packages/pypi.rb new file mode 100644 index 00000000000..fc8a55caa31 --- /dev/null +++ b/app/models/packages/pypi.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true +module Packages + module Pypi + def self.table_name_prefix + 'packages_pypi_' + end + end +end diff --git a/app/models/packages/pypi/metadatum.rb b/app/models/packages/pypi/metadatum.rb new file mode 100644 index 00000000000..7e6456ad964 --- /dev/null +++ b/app/models/packages/pypi/metadatum.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Packages::Pypi::Metadatum < ApplicationRecord + self.primary_key = :package_id + + belongs_to :package, -> { where(package_type: :pypi) }, inverse_of: :pypi_metadatum + + validates :package, presence: true + + validate :pypi_package_type + + private + + def pypi_package_type + unless package&.pypi? + errors.add(:base, _('Package type must be PyPi')) + end + end +end diff --git a/app/models/packages/sem_ver.rb b/app/models/packages/sem_ver.rb new file mode 100644 index 00000000000..b73d51b08b7 --- /dev/null +++ b/app/models/packages/sem_ver.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class Packages::SemVer + attr_accessor :major, :minor, :patch, :prerelease, :build + + def initialize(major = 0, minor = 0, patch = 0, prerelease = nil, build = nil, prefixed: false) + @major = major + @minor = minor + @patch = patch + @prerelease = prerelease + @build = build + @prefixed = prefixed + end + + def prefixed? + @prefixed + end + + def ==(other) + self.class == other.class && + self.major == other.major && + self.minor == other.minor && + self.patch == other.patch && + self.prerelease == other.prerelease && + self.build == other.build + end + + def to_s + s = "#{prefixed? ? 'v' : ''}#{major || 0}.#{minor || 0}.#{patch || 0}" + s += "-#{prerelease}" if prerelease + s += "+#{build}" if build + + s + end + + def self.match(str, prefixed: false) + return unless str&.start_with?('v') == prefixed + + str = str[1..] if prefixed + + Gitlab::Regex.semver_regex.match(str) + end + + def self.match?(str, prefixed: false) + !match(str, prefixed: prefixed).nil? + end + + def self.parse(str, prefixed: false) + m = match str, prefixed: prefixed + return unless m + + new(m[1].to_i, m[2].to_i, m[3].to_i, m[4], m[5], prefixed: prefixed) + end +end diff --git a/app/models/packages/tag.rb b/app/models/packages/tag.rb new file mode 100644 index 00000000000..771d016daed --- /dev/null +++ b/app/models/packages/tag.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +class Packages::Tag < ApplicationRecord + belongs_to :package, inverse_of: :tags + + validates :package, :name, presence: true + + FOR_PACKAGES_TAGS_LIMIT = 200.freeze + NUGET_TAGS_SEPARATOR = ' ' # https://docs.microsoft.com/en-us/nuget/reference/nuspec#tags + + scope :preload_package, -> { preload(:package) } + scope :with_name, -> (name) { where(name: name) } + + def self.for_packages(packages) + where(package_id: packages.select(:id)) + .order(updated_at: :desc) + .limit(FOR_PACKAGES_TAGS_LIMIT) + end +end diff --git a/app/models/performance_monitoring/prometheus_dashboard.rb b/app/models/performance_monitoring/prometheus_dashboard.rb index b04e7e689cd..bf87d2c3916 100644 --- a/app/models/performance_monitoring/prometheus_dashboard.rb +++ b/app/models/performance_monitoring/prometheus_dashboard.rb @@ -7,7 +7,7 @@ module PerformanceMonitoring attr_accessor :dashboard, :panel_groups, :path, :environment, :priority, :templating, :links validates :dashboard, presence: true - validates :panel_groups, presence: true + validates :panel_groups, array_members: { member_class: PerformanceMonitoring::PrometheusPanelGroup } class << self def from_json(json_content) @@ -35,9 +35,15 @@ module PerformanceMonitoring new( dashboard: attributes['dashboard'], - panel_groups: attributes['panel_groups']&.map { |group| PrometheusPanelGroup.from_json(group) } + panel_groups: initialize_children_collection(attributes['panel_groups']) ) end + + def initialize_children_collection(children) + return unless children.is_a?(Array) + + children.map { |group| PerformanceMonitoring::PrometheusPanelGroup.from_json(group) } + end end def to_yaml @@ -47,7 +53,7 @@ module PerformanceMonitoring # This method is planned to be refactored as a part of https://gitlab.com/gitlab-org/gitlab/-/issues/219398 # implementation. For new existing logic was reused to faster deliver MVC def schema_validation_warnings - self.class.from_json(self.as_json) + self.class.from_json(reload_schema) nil rescue ActiveModel::ValidationError => exception exception.model.errors.map { |attr, error| "#{attr}: #{error}" } @@ -55,6 +61,14 @@ module PerformanceMonitoring private + # dashboard finder methods are somehow limited, #find includes checking if + # user is authorised to view selected dashboard, but modifies schema, which in some cases may + # cause false positives returned from validation, and #find_raw does not authorise users + def reload_schema + project = environment&.project + project.nil? ? self.as_json : Gitlab::Metrics::Dashboard::Finder.find_raw(project, dashboard_path: path) + end + def yaml_valid_attributes %w(panel_groups panels metrics group priority type title y_label weight id unit label query query_range dashboard) end diff --git a/app/models/performance_monitoring/prometheus_panel.rb b/app/models/performance_monitoring/prometheus_panel.rb index a16a68ba832..b33c09001ae 100644 --- a/app/models/performance_monitoring/prometheus_panel.rb +++ b/app/models/performance_monitoring/prometheus_panel.rb @@ -7,7 +7,8 @@ module PerformanceMonitoring attr_accessor :type, :title, :y_label, :weight, :metrics, :y_axis, :max_value validates :title, presence: true - validates :metrics, presence: true + validates :metrics, array_members: { member_class: PerformanceMonitoring::PrometheusMetric } + class << self def from_json(json_content) build_from_hash(json_content).tap(&:validate!) @@ -23,9 +24,15 @@ module PerformanceMonitoring title: attributes['title'], y_label: attributes['y_label'], weight: attributes['weight'], - metrics: attributes['metrics']&.map { |metric| PrometheusMetric.from_json(metric) } + metrics: initialize_children_collection(attributes['metrics']) ) end + + def initialize_children_collection(children) + return unless children.is_a?(Array) + + children.map { |metrics| PerformanceMonitoring::PrometheusMetric.from_json(metrics) } + end end def id(group_title) diff --git a/app/models/performance_monitoring/prometheus_panel_group.rb b/app/models/performance_monitoring/prometheus_panel_group.rb index f88106f259b..7f3d2a1b8f4 100644 --- a/app/models/performance_monitoring/prometheus_panel_group.rb +++ b/app/models/performance_monitoring/prometheus_panel_group.rb @@ -7,7 +7,8 @@ module PerformanceMonitoring attr_accessor :group, :priority, :panels validates :group, presence: true - validates :panels, presence: true + validates :panels, array_members: { member_class: PerformanceMonitoring::PrometheusPanel } + class << self def from_json(json_content) build_from_hash(json_content).tap(&:validate!) @@ -21,9 +22,15 @@ module PerformanceMonitoring new( group: attributes['group'], priority: attributes['priority'], - panels: attributes['panels']&.map { |panel| PrometheusPanel.from_json(panel) } + panels: initialize_children_collection(attributes['panels']) ) end + + def initialize_children_collection(children) + return unless children.is_a?(Array) + + children.map { |panels| PerformanceMonitoring::PrometheusPanel.from_json(panels) } + end end end end diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 7afee2a35cb..488ebd531a8 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -17,11 +17,13 @@ class PersonalAccessToken < ApplicationRecord before_save :ensure_token - scope :active, -> { where("revoked = false AND (expires_at >= NOW() OR expires_at IS NULL)") } - scope :expiring_and_not_notified, ->(date) { where(["revoked = false AND expire_notification_delivered = false AND expires_at >= NOW() AND expires_at <= ?", date]) } - scope :inactive, -> { where("revoked = true OR expires_at < NOW()") } + scope :active, -> { where("revoked = false AND (expires_at >= CURRENT_DATE OR expires_at IS NULL)") } + scope :expiring_and_not_notified, ->(date) { where(["revoked = false AND expire_notification_delivered = false AND expires_at >= CURRENT_DATE AND expires_at <= ?", date]) } + scope :inactive, -> { where("revoked = true OR expires_at < CURRENT_DATE") } scope :with_impersonation, -> { where(impersonation: true) } scope :without_impersonation, -> { where(impersonation: false) } + scope :revoked, -> { where(revoked: true) } + scope :not_revoked, -> { where(revoked: [false, nil]) } scope :for_user, -> (user) { where(user: user) } scope :preload_users, -> { preload(:user) } scope :order_expires_at_asc, -> { reorder(expires_at: :asc) } diff --git a/app/models/plan.rb b/app/models/plan.rb index acac5f9aeae..b4091e0a755 100644 --- a/app/models/plan.rb +++ b/app/models/plan.rb @@ -27,7 +27,7 @@ class Plan < ApplicationRecord end def actual_limits - self.limits || PlanLimits.new + self.limits || self.build_limits end def default? diff --git a/app/models/plan_limits.rb b/app/models/plan_limits.rb index 575105cfd79..f17078c0cab 100644 --- a/app/models/plan_limits.rb +++ b/app/models/plan_limits.rb @@ -1,23 +1,36 @@ # frozen_string_literal: true class PlanLimits < ApplicationRecord + LimitUndefinedError = Class.new(StandardError) + belongs_to :plan - def exceeded?(limit_name, object) - return false unless enabled?(limit_name) + def exceeded?(limit_name, subject, alternate_limit: 0) + limit = limit_for(limit_name, alternate_limit: alternate_limit) + return false unless limit - if object.is_a?(Integer) - object >= read_attribute(limit_name) - else - # object.count >= limit value is slower than checking + case subject + when Integer + subject >= limit + when ActiveRecord::Relation + # We intentionally not accept just plain ApplicationRecord classes to + # enforce the subject to be scoped down to a relation first. + # + # subject.count >= limit value is slower than checking # if a record exists at the limit value - 1 position. - object.offset(read_attribute(limit_name) - 1).exists? + subject.offset(limit - 1).exists? + else + raise ArgumentError, "#{subject.class} is not supported as a limit value" end end - private + def limit_for(limit_name, alternate_limit: 0) + limit = read_attribute(limit_name) + raise LimitUndefinedError, "The limit `#{limit_name}` is undefined" if limit.nil? + + alternate_limit = alternate_limit.call if alternate_limit.respond_to?(:call) - def enabled?(limit_name) - read_attribute(limit_name) > 0 + limits = [limit, alternate_limit] + limits.map(&:to_i).select(&:positive?).min end end diff --git a/app/models/product_analytics_event.rb b/app/models/product_analytics_event.rb new file mode 100644 index 00000000000..95a2e7a26c4 --- /dev/null +++ b/app/models/product_analytics_event.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class ProductAnalyticsEvent < ApplicationRecord + self.table_name = 'product_analytics_events_experimental' + + # Ignore that the partition key :project_id is part of the formal primary key + self.primary_key = :id + + belongs_to :project + + validates :event_id, :project_id, :v_collector, :v_etl, presence: true + + # There is no default Rails timestamps in the table. + # collector_tstamp is a timestamp when a collector recorded an event. + scope :order_by_time, -> { order(collector_tstamp: :desc) } + + # If we decide to change this scope to use date_trunc('day', collector_tstamp), + # we should remember that a btree index on collector_tstamp will be no longer effective. + scope :timerange, ->(duration, today = Time.zone.today) { + where('collector_tstamp BETWEEN ? AND ? ', today - duration + 1, today + 1) + } +end diff --git a/app/models/project.rb b/app/models/project.rb index 845e9e83e78..3aa0db56404 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -65,6 +65,7 @@ class Project < ApplicationRecord cache_markdown_field :description, pipeline: :description + default_value_for :packages_enabled, true default_value_for :archived, false default_value_for :resolve_outdated_diff_discussions, false default_value_for :container_registry_enabled, gitlab_config_features.container_registry @@ -168,6 +169,7 @@ class Project < ApplicationRecord has_one :custom_issue_tracker_service has_one :bugzilla_service has_one :gitlab_issue_tracker_service, inverse_of: :project + has_one :confluence_service has_one :external_wiki_service has_one :prometheus_service, inverse_of: :project has_one :mock_ci_service @@ -190,6 +192,10 @@ class Project < ApplicationRecord has_many :forks, through: :forked_to_members, source: :project, inverse_of: :forked_from_project has_many :fork_network_projects, through: :fork_network, source: :projects + # Packages + has_many :packages, class_name: 'Packages::Package' + has_many :package_files, through: :packages, class_name: 'Packages::PackageFile' + has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :export_jobs, class_name: 'ProjectExportJob' @@ -200,6 +206,7 @@ class Project < ApplicationRecord has_one :grafana_integration, inverse_of: :project has_one :project_setting, inverse_of: :project, autosave: true has_one :alerting_setting, inverse_of: :project, class_name: 'Alerting::ProjectAlertingSetting' + has_one :service_desk_setting, class_name: 'ServiceDeskSetting' # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id', inverse_of: :target_project @@ -363,6 +370,7 @@ class Project < ApplicationRecord to: :project_setting, allow_nil: true delegate :scheduled?, :started?, :in_progress?, :failed?, :finished?, prefix: :import, to: :import_state, allow_nil: true + delegate :squash_always?, :squash_never?, :squash_enabled_by_default?, :squash_readonly?, to: :project_setting delegate :no_import?, to: :import_state, allow_nil: true delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true @@ -376,7 +384,10 @@ class Project < ApplicationRecord delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings, prefix: :ci delegate :forward_deployment_enabled, :forward_deployment_enabled=, :forward_deployment_enabled?, to: :ci_cd_settings delegate :actual_limits, :actual_plan_name, to: :namespace, allow_nil: true - delegate :allow_merge_on_skipped_pipeline, :allow_merge_on_skipped_pipeline?, :allow_merge_on_skipped_pipeline=, to: :project_setting + delegate :allow_merge_on_skipped_pipeline, :allow_merge_on_skipped_pipeline?, + :allow_merge_on_skipped_pipeline=, :has_confluence?, + to: :project_setting + delegate :active?, to: :prometheus_service, allow_nil: true, prefix: true # Validations validates :creator, presence: true, on: :create @@ -439,6 +450,7 @@ class Project < ApplicationRecord # Sometimes queries (e.g. using CTEs) require explicit disambiguation with table name scope :projects_order_id_desc, -> { reorder(self.arel_table['id'].desc) } + scope :with_packages, -> { joins(:packages) } scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) } scope :personal, ->(user) { where(namespace_id: user.namespace_id) } scope :joined, ->(user) { where('namespace_id != ?', user.namespace_id) } @@ -454,6 +466,7 @@ class Project < ApplicationRecord scope :with_statistics, -> { includes(:statistics) } scope :with_namespace, -> { includes(:namespace) } scope :with_import_state, -> { includes(:import_state) } + scope :include_project_feature, -> { includes(:project_feature) } scope :with_service, ->(service) { joins(service).eager_load(service) } scope :with_shared_runners, -> { where(shared_runners_enabled: true) } scope :with_container_registry, -> { where(container_registry_enabled: true) } @@ -488,6 +501,7 @@ class Project < ApplicationRecord .where(repository_languages: { programming_language_id: lang_id_query }) end + scope :service_desk_enabled, -> { where(service_desk_enabled: true) } scope :with_builds_enabled, -> { with_feature_enabled(:builds) } scope :with_issues_enabled, -> { with_feature_enabled(:issues) } scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) } @@ -513,9 +527,8 @@ class Project < ApplicationRecord .where(project_pages_metadata: { project_id: nil }) end - scope :with_api_entity_associations, -> { - preload(:project_feature, :route, :tags, - group: :ip_restrictions, namespace: [:route, :owner]) + scope :with_api_commit_entity_associations, -> { + preload(:project_feature, :route, namespace: [:route, :owner]) } enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } @@ -532,6 +545,10 @@ class Project < ApplicationRecord # Used by Projects::CleanupService to hold a map of rewritten object IDs mount_uploader :bfg_object_map, AttachmentUploader + def self.with_api_entity_associations + preload(:project_feature, :route, :tags, :group, namespace: [:route, :owner]) + end + def self.with_web_entity_associations preload(:project_feature, :route, :creator, :group, namespace: [:route, :owner]) end @@ -589,6 +606,14 @@ class Project < ApplicationRecord end end + def self.projects_user_can(projects, user, action) + projects = where(id: projects) + + DeclarativePolicy.user_scope do + projects.select { |project| Ability.allowed?(user, action, project) } + end + end + # This scope returns projects where user has access to both the project and the feature. def self.filter_by_feature_visibility(feature, user) with_feature_available_for_user(feature, user) @@ -675,10 +700,11 @@ class Project < ApplicationRecord # '>' or its escaped form ('>') are checked for because '>' is sometimes escaped # when the reference comes from an external source. def markdown_reference_pattern - %r{ - #{reference_pattern} - (#{reference_postfix}|#{reference_postfix_escaped}) - }x + @markdown_reference_pattern ||= + %r{ + #{reference_pattern} + (#{reference_postfix}|#{reference_postfix_escaped}) + }x end def trending @@ -706,6 +732,12 @@ class Project < ApplicationRecord from_union([with_issues_enabled, with_merge_requests_enabled]).select(:id) end + + def find_by_service_desk_project_key(key) + # project_key is not indexed for now + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24063#note_282435524 for details + joins(:service_desk_setting).find_by('service_desk_settings.project_key' => key) + end end def initialize(attributes = nil) @@ -839,6 +871,15 @@ class Project < ApplicationRecord end end + # Because we use default_value_for we need to be sure + # packages_enabled= method does exist even if we rollback migration. + # Otherwise many tests from spec/migrations will fail. + def packages_enabled=(value) + if has_attribute?(:packages_enabled) + write_attribute(:packages_enabled, value) + end + end + def cleanup @repository = nil end @@ -1699,7 +1740,7 @@ class Project < ApplicationRecord url_path = full_path.partition('/').last # If the project path is the same as host, we serve it as group page - return url if url == "#{Settings.pages.protocol}://#{url_path}" + return url if url == "#{Settings.pages.protocol}://#{url_path}".downcase "#{url}/#{url_path}" end @@ -1795,6 +1836,7 @@ class Project < ApplicationRecord after_create_default_branch join_pool_repository refresh_markdown_cache! + write_repository_config end def update_project_counter_caches @@ -1922,6 +1964,7 @@ class Project < ApplicationRecord .append(key: 'CI_PROJECT_PATH', value: full_path) .append(key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug) .append(key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path) + .append(key: 'CI_PROJECT_ROOT_NAMESPACE', value: namespace.root_ancestor.path) .append(key: 'CI_PROJECT_URL', value: web_url) .append(key: 'CI_PROJECT_VISIBILITY', value: Gitlab::VisibilityLevel.string_level(visibility_level)) .append(key: 'CI_PROJECT_REPOSITORY_LANGUAGES', value: repository_languages.map(&:name).join(',').downcase) @@ -2131,7 +2174,13 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def forks_count - Projects::ForksCountService.new(self).count + BatchLoader.for(self).batch do |projects, loader| + fork_count_per_project = ::Projects::BatchForksCountService.new(projects).refresh_cache_and_retrieve_data + + fork_count_per_project.each do |project, count| + loader.call(project, count) + end + end end # rubocop: enable CodeReuse/ServiceClass @@ -2410,6 +2459,37 @@ class Project < ApplicationRecord super || build_metrics_setting end + def service_desk_enabled + Gitlab::ServiceDesk.enabled?(project: self) + end + + alias_method :service_desk_enabled?, :service_desk_enabled + + def service_desk_address + return unless service_desk_enabled? + + config = Gitlab.config.incoming_email + wildcard = Gitlab::IncomingEmail::WILDCARD_PLACEHOLDER + + config.address&.gsub(wildcard, "#{full_path_slug}-#{id}-issue-") + end + + def root_namespace + if namespace.has_parent? + namespace.root_ancestor + else + namespace + end + end + + def package_already_taken?(package_name) + namespace.root_ancestor.all_projects + .joins(:packages) + .where.not(id: id) + .merge(Packages::Package.with_name(package_name)) + .exists? + end + private def find_service(services, name) diff --git a/app/models/project_services/alerts_service.rb b/app/models/project_services/alerts_service.rb index 58c47accfd1..28902114f3c 100644 --- a/app/models/project_services/alerts_service.rb +++ b/app/models/project_services/alerts_service.rb @@ -78,3 +78,5 @@ class AlertsService < Service Gitlab::Routing.url_helpers end end + +AlertsService.prepend_if_ee('EE::AlertsService') diff --git a/app/models/project_services/bugzilla_service.rb b/app/models/project_services/bugzilla_service.rb index 0a498fde95a..4332db3e961 100644 --- a/app/models/project_services/bugzilla_service.rb +++ b/app/models/project_services/bugzilla_service.rb @@ -3,11 +3,11 @@ class BugzillaService < IssueTrackerService validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - def default_title + def title 'Bugzilla' end - def default_description + def description s_('IssueTracker|Bugzilla issue tracker') end diff --git a/app/models/project_services/confluence_service.rb b/app/models/project_services/confluence_service.rb new file mode 100644 index 00000000000..dd44a0d1d56 --- /dev/null +++ b/app/models/project_services/confluence_service.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +class ConfluenceService < Service + include ActionView::Helpers::UrlHelper + + VALID_SCHEME_MATCH = %r{\Ahttps?\Z}.freeze + VALID_HOST_MATCH = %r{\A.+\.atlassian\.net\Z}.freeze + VALID_PATH_MATCH = %r{\A/wiki(/|\Z)}.freeze + + prop_accessor :confluence_url + + validates :confluence_url, presence: true, if: :activated? + validate :validate_confluence_url_is_cloud, if: :activated? + + after_commit :cache_project_has_confluence + + def self.to_param + 'confluence' + end + + def self.supported_events + %w() + end + + def title + s_('ConfluenceService|Confluence Workspace') + end + + def description + s_('ConfluenceService|Connect a Confluence Cloud Workspace to your GitLab project') + end + + def detailed_description + return unless project.wiki_enabled? + + if activated? + wiki_url = project.wiki.web_url + + s_( + 'ConfluenceService|Your GitLab Wiki can be accessed here: %{wiki_link}. To re-enable your GitLab Wiki, disable this integration' % + { wiki_link: link_to(wiki_url, wiki_url) } + ).html_safe + else + s_('ConfluenceService|Enabling the Confluence Workspace will disable the default GitLab Wiki. Your GitLab Wiki data will be saved and you can always re-enable it later by turning off this integration').html_safe + end + end + + def fields + [ + { + type: 'text', + name: 'confluence_url', + title: 'Confluence Cloud Workspace URL', + placeholder: s_('ConfluenceService|The URL of the Confluence Workspace'), + required: true + } + ] + end + + def can_test? + false + end + + private + + def validate_confluence_url_is_cloud + unless confluence_uri_valid? + errors.add(:confluence_url, 'URL must be to a Confluence Cloud Workspace hosted on atlassian.net') + end + end + + def confluence_uri_valid? + return false unless confluence_url + + uri = URI.parse(confluence_url) + + (uri.scheme&.match(VALID_SCHEME_MATCH) && + uri.host&.match(VALID_HOST_MATCH) && + uri.path&.match(VALID_PATH_MATCH)).present? + + rescue URI::InvalidURIError + false + end + + def cache_project_has_confluence + return unless project && !project.destroyed? + + project.project_setting.save! unless project.project_setting.persisted? + project.project_setting.update_column(:has_confluence, active?) + end +end diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb index dbc42b1b86d..fc58ba27c3d 100644 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ b/app/models/project_services/custom_issue_tracker_service.rb @@ -3,11 +3,11 @@ class CustomIssueTrackerService < IssueTrackerService validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - def default_title + def title 'Custom Issue Tracker' end - def default_description + def description s_('IssueTracker|Custom issue tracker') end @@ -17,8 +17,6 @@ class CustomIssueTrackerService < IssueTrackerService def fields [ - { type: 'text', name: 'title', placeholder: title }, - { type: 'text', name: 'description', placeholder: description }, { type: 'text', name: 'project_url', placeholder: 'Project url', required: true }, { type: 'text', name: 'issues_url', placeholder: 'Issue url', required: true }, { type: 'text', name: 'new_issue_url', placeholder: 'New Issue url', required: true } diff --git a/app/models/project_services/gitlab_issue_tracker_service.rb b/app/models/project_services/gitlab_issue_tracker_service.rb index ec28602b5e6..b3f44e040bc 100644 --- a/app/models/project_services/gitlab_issue_tracker_service.rb +++ b/app/models/project_services/gitlab_issue_tracker_service.rb @@ -7,11 +7,11 @@ class GitlabIssueTrackerService < IssueTrackerService default_value_for :default, true - def default_title + def title 'GitLab' end - def default_description + def description s_('IssueTracker|GitLab issue tracker') end diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index f5d6ae10469..694374e9548 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -25,28 +25,6 @@ class IssueTrackerService < Service end end - # this will be removed as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - def title - if title_attribute = read_attribute(:title) - title_attribute - elsif self.properties && self.properties['title'].present? - self.properties['title'] - else - default_title - end - end - - # this will be removed as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - def description - if description_attribute = read_attribute(:description) - description_attribute - elsif self.properties && self.properties['description'].present? - self.properties['description'] - else - default_description - end - end - def handle_properties # this has been moved from initialize_properties and should be improved # as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 @@ -54,13 +32,6 @@ class IssueTrackerService < Service @legacy_properties_data = properties.dup data_values = properties.slice!('title', 'description') - properties.each do |key, _| - current_value = self.properties.delete(key) - value = attribute_changed?(key) ? attribute_change(key).last : current_value - - write_attribute(key, value) - end - data_values.reject! { |key| data_fields.changed.include?(key) } data_values.slice!(*data_fields.attributes.keys) data_fields.assign_attributes(data_values) if data_values.present? @@ -102,7 +73,6 @@ class IssueTrackerService < Service def fields [ - { type: 'text', name: 'description', placeholder: description }, { type: 'text', name: 'project_url', placeholder: 'Project url', required: true }, { type: 'text', name: 'issues_url', placeholder: 'Issue url', required: true }, { type: 'text', name: 'new_issue_url', placeholder: 'New Issue url', required: true } @@ -117,8 +87,6 @@ class IssueTrackerService < Service def set_default_data return unless issues_tracker.present? - self.title ||= issues_tracker['title'] - # we don't want to override if we have set something return if project_url || issues_url || new_issue_url diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index bb4d35cad22..4ea2ec10f11 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -23,7 +23,7 @@ class JiraService < IssueTrackerService # TODO: we can probably just delegate as part of # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :username, :password, :url, :api_url, :jira_issue_transition_id + data_field :username, :password, :url, :api_url, :jira_issue_transition_id, :project_key, :issues_enabled before_update :reset_password @@ -64,8 +64,6 @@ class JiraService < IssueTrackerService def set_default_data return unless issues_tracker.present? - self.title ||= issues_tracker['title'] - return if url data_fields.url ||= issues_tracker['url'] @@ -103,11 +101,11 @@ class JiraService < IssueTrackerService [Jira service documentation](#{help_page_url('user/project/integrations/jira')})." end - def default_title + def title 'Jira' end - def default_description + def description s_('JiraService|Jira issue tracker') end @@ -130,7 +128,7 @@ class JiraService < IssueTrackerService end def new_issue_url - "#{url}/secure/CreateIssue.jspa" + "#{url}/secure/CreateIssue!default.jspa" end alias_method :original_url, :url @@ -442,3 +440,5 @@ class JiraService < IssueTrackerService end end end + +JiraService.prepend_if_ee('EE::JiraService') diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 44a41969b1c..997c6eba91a 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -28,6 +28,9 @@ class PrometheusService < MonitoringService after_create_commit :create_default_alerts + scope :preload_project, -> { preload(:project) } + scope :with_clusters_with_cilium, -> { joins(project: [:clusters]).merge(Clusters::Cluster.with_available_cilium) } + def initialize_properties if properties.nil? self.properties = {} @@ -51,7 +54,7 @@ class PrometheusService < MonitoringService end def fields - result = [ + [ { type: 'checkbox', name: 'manual_configuration', @@ -64,30 +67,23 @@ class PrometheusService < MonitoringService title: 'API URL', placeholder: s_('PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/'), required: true + }, + { + type: 'text', + name: 'google_iap_audience_client_id', + title: 'Google IAP Audience Client ID', + placeholder: s_('PrometheusService|Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)'), + autocomplete: 'off', + required: false + }, + { + type: 'textarea', + name: 'google_iap_service_account_json', + title: 'Google IAP Service Account JSON', + placeholder: s_('PrometheusService|Contents of the credentials.json file of your service account, like: { "type": "service_account", "project_id": ... }'), + required: false } ] - - if Feature.enabled?(:prometheus_service_iap_auth) - result += [ - { - type: 'text', - name: 'google_iap_audience_client_id', - title: 'Google IAP Audience Client ID', - placeholder: s_('PrometheusService|Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)'), - autocomplete: 'off', - required: false - }, - { - type: 'textarea', - name: 'google_iap_service_account_json', - title: 'Google IAP Service Account JSON', - placeholder: s_('PrometheusService|Contents of the credentials.json file of your service account, like: { "type": "service_account", "project_id": ... }'), - required: false - } - ] - end - - result end # Check we can connect to the Prometheus API @@ -103,7 +99,7 @@ class PrometheusService < MonitoringService options = { allow_local_requests: allow_local_api_url? } - if Feature.enabled?(:prometheus_service_iap_auth) && behind_iap? + if behind_iap? # Adds the Authorization header options[:headers] = iap_client.apply({}) end diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb index a4ca0d20669..df78520d65f 100644 --- a/app/models/project_services/redmine_service.rb +++ b/app/models/project_services/redmine_service.rb @@ -3,11 +3,11 @@ class RedmineService < IssueTrackerService validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - def default_title + def title 'Redmine' end - def default_description + def description s_('IssueTracker|Redmine issue tracker') end diff --git a/app/models/project_services/youtrack_service.rb b/app/models/project_services/youtrack_service.rb index 40203ad692d..7fb3bde44a5 100644 --- a/app/models/project_services/youtrack_service.rb +++ b/app/models/project_services/youtrack_service.rb @@ -12,11 +12,11 @@ class YoutrackService < IssueTrackerService end end - def default_title + def title 'YouTrack' end - def default_description + def description s_('IssueTracker|YouTrack issue tracker') end @@ -26,7 +26,6 @@ class YoutrackService < IssueTrackerService def fields [ - { type: 'text', name: 'description', placeholder: description }, { type: 'text', name: 'project_url', title: 'Project URL', placeholder: 'Project URL', required: true }, { type: 'text', name: 'issues_url', title: 'Issue URL', placeholder: 'Issue URL', required: true } ] diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index 9022d3e879d..aca7eec3382 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -3,7 +3,22 @@ class ProjectSetting < ApplicationRecord belongs_to :project, inverse_of: :project_setting + enum squash_option: { + never: 0, + always: 1, + default_on: 2, + default_off: 3 + }, _prefix: 'squash' + self.primary_key = :project_id + + def squash_enabled_by_default? + %w[always default_on].include?(squash_option) + end + + def squash_readonly? + %w[always never].include?(squash_option) + end end ProjectSetting.prepend_if_ee('EE::ProjectSetting') diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 6f04a36392d..f153bfe3f5b 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -7,16 +7,12 @@ class ProjectStatistics < ApplicationRecord belongs_to :namespace default_value_for :wiki_size, 0 - - # older migrations fail due to non-existent attribute without this - def wiki_size - has_attribute?(:wiki_size) ? super : 0 - end + default_value_for :snippets_size, 0 before_save :update_storage_size - COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count].freeze - INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size] }.freeze + COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size].freeze + INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size], snippets_size: %i[storage_size] }.freeze NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size].freeze scope :for_project_ids, ->(project_ids) { where(project_id: project_ids) } @@ -54,17 +50,37 @@ class ProjectStatistics < ApplicationRecord self.wiki_size = project.wiki.repository.size * 1.megabyte end + def update_snippets_size + self.snippets_size = project.snippets.with_statistics.sum(:repository_size) + end + def update_lfs_objects_size self.lfs_objects_size = project.lfs_objects.sum(:size) end - # older migrations fail due to non-existent attribute without this - def packages_size - has_attribute?(:packages_size) ? super : 0 + # `wiki_size` and `snippets_size` have no default value in the database + # and the column can be nil. + # This means that, when the columns were added, all rows had nil + # values on them. + # Therefore, any call to any of those methods will return nil instead + # of 0, because `default_value_for` works with new records, not existing ones. + # + # These two methods provide consistency and avoid returning nil. + def wiki_size + super.to_i + end + + def snippets_size + super.to_i end def update_storage_size - self.storage_size = repository_size + wiki_size.to_i + lfs_objects_size + build_artifacts_size + packages_size + storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size + # The `snippets_size` column was added on 20200622095419 but db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb + # might try to update project statistics before the `snippets_size` column has been created. + storage_size += snippets_size if self.class.column_names.include?('snippets_size') + + self.storage_size = storage_size end # Since this incremental update method does not call update_storage_size above, diff --git a/app/models/prometheus_alert.rb b/app/models/prometheus_alert.rb index fbc0281296f..32f9809e538 100644 --- a/app/models/prometheus_alert.rb +++ b/app/models/prometheus_alert.rb @@ -16,6 +16,7 @@ class PrometheusAlert < ApplicationRecord has_many :prometheus_alert_events, inverse_of: :prometheus_alert has_many :related_issues, through: :prometheus_alert_events + has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :prometheus_alert after_save :clear_prometheus_adapter_cache! after_destroy :clear_prometheus_adapter_cache! diff --git a/app/models/prometheus_metric.rb b/app/models/prometheus_metric.rb index 571b586056b..bfd23d2a334 100644 --- a/app/models/prometheus_metric.rb +++ b/app/models/prometheus_metric.rb @@ -11,6 +11,7 @@ class PrometheusMetric < ApplicationRecord validates :group, presence: true validates :y_label, presence: true validates :unit, presence: true + validates :identifier, uniqueness: { scope: :project_id }, allow_nil: true validates :project, presence: true, unless: :common? validates :project, absence: true, if: :common? diff --git a/app/models/repository.rb b/app/models/repository.rb index 911cfc7db38..48e96d4c193 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -149,7 +149,8 @@ class Repository before: opts[:before], all: !!opts[:all], first_parent: !!opts[:first_parent], - order: opts[:order] + order: opts[:order], + literal_pathspec: opts.fetch(:literal_pathspec, true) } commits = Gitlab::Git::Commit.where(options) @@ -676,24 +677,24 @@ class Repository end end - def list_last_commits_for_tree(sha, path, offset: 0, limit: 25) - commits = raw_repository.list_last_commits_for_tree(sha, path, offset: offset, limit: limit) + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25, literal_pathspec: false) + commits = raw_repository.list_last_commits_for_tree(sha, path, offset: offset, limit: limit, literal_pathspec: literal_pathspec) commits.each do |path, commit| commits[path] = ::Commit.new(commit, container) end end - def last_commit_for_path(sha, path) - commit = raw_repository.last_commit_for_path(sha, path) + def last_commit_for_path(sha, path, literal_pathspec: false) + commit = raw_repository.last_commit_for_path(sha, path, literal_pathspec: literal_pathspec) ::Commit.new(commit, container) if commit end - def last_commit_id_for_path(sha, path) + def last_commit_id_for_path(sha, path, literal_pathspec: false) key = path.blank? ? "last_commit_id_for_path:#{sha}" : "last_commit_id_for_path:#{sha}:#{Digest::SHA1.hexdigest(path)}" cache.fetch(key) do - last_commit_for_path(sha, path)&.id + last_commit_for_path(sha, path, literal_pathspec: literal_pathspec)&.id end end @@ -712,8 +713,8 @@ class Repository "#{name}-#{highest_branch_id + 1}" end - def branches_sorted_by(value) - raw_repository.local_branches(sort_by: value) + def branches_sorted_by(sort_by, pagination_params = nil) + raw_repository.local_branches(sort_by: sort_by, pagination_params: pagination_params) end def tags_sorted_by(value) @@ -1113,7 +1114,7 @@ class Repository def project if repo_type.snippet? container.project - else + elsif container.is_a?(Project) container end end diff --git a/app/models/resource_event.rb b/app/models/resource_event.rb index 86e11c2d568..26dcda2630a 100644 --- a/app/models/resource_event.rb +++ b/app/models/resource_event.rb @@ -11,6 +11,7 @@ class ResourceEvent < ApplicationRecord belongs_to :user scope :created_after, ->(time) { where('created_at > ?', time) } + scope :created_on_or_before, ->(time) { where('created_at <= ?', time) } def discussion_id strong_memoize(:discussion_id) do diff --git a/app/models/resource_state_event.rb b/app/models/resource_state_event.rb index 1d6573b180f..766b4d7a865 100644 --- a/app/models/resource_state_event.rb +++ b/app/models/resource_state_event.rb @@ -6,10 +6,16 @@ class ResourceStateEvent < ResourceEvent validate :exactly_one_issuable + belongs_to :source_merge_request, class_name: 'MergeRequest', foreign_key: :source_merge_request_id + # state is used for issue and merge request states. enum state: Issue.available_states.merge(MergeRequest.available_states).merge(reopened: 5) def self.issuable_attrs %i(issue merge_request).freeze end + + def issuable + issue || merge_request + end end diff --git a/app/models/service.rb b/app/models/service.rb index 2880526c9de..89bde61bfe1 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -7,9 +7,12 @@ class Service < ApplicationRecord include Importable include ProjectServicesLoggable include DataFields + include IgnorableColumns + + ignore_columns %i[title description], remove_with: '13.4', remove_after: '2020-09-22' SERVICE_NAMES = %w[ - alerts asana assembla bamboo bugzilla buildkite campfire custom_issue_tracker discord + alerts asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker discord drone_ci emails_on_push external_wiki flowdock hangouts_chat hipchat irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack @@ -357,6 +360,14 @@ class Service < ApplicationRecord service end + def self.instance_exists_for?(type) + exists?(instance: true, type: type) + end + + def self.instance_for(type) + find_by(instance: true, type: type) + end + # override if needed def supports_data_fields? false @@ -381,30 +392,7 @@ class Service < ApplicationRecord end def self.event_description(event) - case event - when "push", "push_events" - "Event will be triggered by a push to the repository" - when "tag_push", "tag_push_events" - "Event will be triggered when a new tag is pushed to the repository" - when "note", "note_events" - "Event will be triggered when someone adds a comment" - when "issue", "issue_events" - "Event will be triggered when an issue is created/updated/closed" - when "confidential_issue", "confidential_issue_events" - "Event will be triggered when a confidential issue is created/updated/closed" - when "merge_request", "merge_request_events" - "Event will be triggered when a merge request is created/updated/merged" - when "pipeline", "pipeline_events" - "Event will be triggered when a pipeline status changes" - when "wiki_page", "wiki_page_events" - "Event will be triggered when a wiki page is created/updated" - when "commit", "commit_events" - "Event will be triggered when a commit is created/updated" - when "deployment" - "Event will be triggered when a deployment finishes" - when "alert" - "Event will be triggered when a new, unique alert is recorded" - end + ServicesHelper.service_event_description(event) end def valid_recipients? diff --git a/app/models/service_desk_setting.rb b/app/models/service_desk_setting.rb new file mode 100644 index 00000000000..bcc17d32272 --- /dev/null +++ b/app/models/service_desk_setting.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ServiceDeskSetting < ApplicationRecord + include Gitlab::Utils::StrongMemoize + + belongs_to :project + validates :project_id, presence: true + validate :valid_issue_template + validates :outgoing_name, length: { maximum: 255 }, allow_blank: true + validates :project_key, length: { maximum: 255 }, allow_blank: true, format: { with: /\A[a-z0-9_]+\z/ } + + def issue_template_content + strong_memoize(:issue_template_content) do + next unless issue_template_key.present? + + Gitlab::Template::IssueTemplate.find(issue_template_key, project).content + rescue ::Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError + end + end + + def issue_template_missing? + issue_template_key.present? && !issue_template_content.present? + end + + def valid_issue_template + if issue_template_missing? + errors.add(:issue_template_key, 'is empty or does not exist') + end + end +end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index b63ab003711..eb3960ff12b 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -45,6 +45,9 @@ class Snippet < ApplicationRecord has_many :user_mentions, class_name: "SnippetUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_one :snippet_repository, inverse_of: :snippet + # We need to add the `dependent` in order to call the after_destroy callback + has_one :statistics, class_name: 'SnippetStatistics', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + delegate :name, :email, to: :author, prefix: true, allow_nil: true validates :author, presence: true @@ -68,6 +71,7 @@ class Snippet < ApplicationRecord validates :visibility_level, inclusion: { in: Gitlab::VisibilityLevel.values } after_save :store_mentions!, if: :any_mentionable_attributes_changed? + after_create :create_statistics # Scopes scope :are_internal, -> { where(visibility_level: Snippet::INTERNAL) } @@ -77,6 +81,7 @@ class Snippet < ApplicationRecord scope :fresh, -> { order("created_at DESC") } scope :inc_author, -> { includes(:author) } scope :inc_relations_for_view, -> { includes(author: :status) } + scope :with_statistics, -> { joins(:statistics) } attr_mentionable :description @@ -331,7 +336,13 @@ class Snippet < ApplicationRecord def file_name_on_repo return if repository.empty? - repository.ls_files(repository.root_ref).first + list_files(repository.root_ref).first + end + + def list_files(ref = nil) + return [] if repository.empty? + + repository.ls_files(ref) end class << self diff --git a/app/models/snippet_input_action.rb b/app/models/snippet_input_action.rb index 7f4ab775ab0..cc6373264cc 100644 --- a/app/models/snippet_input_action.rb +++ b/app/models/snippet_input_action.rb @@ -15,9 +15,10 @@ class SnippetInputAction validates :action, inclusion: { in: ACTIONS, message: "%{value} is not a valid action" } validates :previous_path, presence: true, if: :move_action? - validates :file_path, presence: true + validates :file_path, presence: true, unless: :create_action? validates :content, presence: true, if: -> (action) { action.create_action? || action.update_action? } validate :ensure_same_file_path_and_previous_path, if: :update_action? + validate :ensure_different_file_path_and_previous_path, if: :move_action? validate :ensure_allowed_action def initialize(action: nil, previous_path: nil, file_path: nil, content: nil, allowed_actions: nil) @@ -52,6 +53,12 @@ class SnippetInputAction errors.add(:file_path, "can't be different from the previous_path attribute") end + def ensure_different_file_path_and_previous_path + return if previous_path != file_path + + errors.add(:file_path, 'must be different from the previous_path attribute') + end + def ensure_allowed_action return if @allowed_actions.empty? diff --git a/app/models/snippet_statistics.rb b/app/models/snippet_statistics.rb new file mode 100644 index 00000000000..7439f98d114 --- /dev/null +++ b/app/models/snippet_statistics.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class SnippetStatistics < ApplicationRecord + include AfterCommitQueue + include UpdateProjectStatistics + + belongs_to :snippet + + validates :snippet, presence: true + + update_project_statistics project_statistics_name: :snippets_size, statistic_attribute: :repository_size + + delegate :repository, :project, :project_id, to: :snippet + + after_save :update_author_root_storage_statistics, if: :update_author_root_storage_statistics? + after_destroy :update_author_root_storage_statistics, unless: :project_snippet? + + def update_commit_count + self.commit_count = repository.commit_count + end + + def update_repository_size + self.repository_size = repository.size.megabytes + end + + def update_file_count + count = if snippet.repository_exists? + repository.ls_files(repository.root_ref).size + else + 0 + end + + self.file_count = count + end + + def refresh! + update_commit_count + update_repository_size + update_file_count + + save! + end + + private + + alias_method :original_update_project_statistics_after_save?, :update_project_statistics_after_save? + def update_project_statistics_after_save? + project_snippet? && original_update_project_statistics_after_save? + end + + alias_method :original_update_project_statistics_after_destroy?, :update_project_statistics_after_destroy? + def update_project_statistics_after_destroy? + project_snippet? && original_update_project_statistics_after_destroy? + end + + def update_author_root_storage_statistics? + !project_snippet? && saved_change_to_repository_size? + end + + def update_author_root_storage_statistics + run_after_commit do + Namespaces::ScheduleAggregationWorker.perform_async(snippet.author.namespace_id) + end + end + + def project_snippet? + snippet.is_a?(ProjectSnippet) + end +end diff --git a/app/models/state_note.rb b/app/models/state_note.rb index cbcb1c2b49d..5e35f15aac4 100644 --- a/app/models/state_note.rb +++ b/app/models/state_note.rb @@ -1,19 +1,47 @@ # frozen_string_literal: true class StateNote < SyntheticNote + include Gitlab::Utils::StrongMemoize + def self.from_event(event, resource: nil, resource_parent: nil) - attrs = note_attributes(event.state, event, resource, resource_parent) + attrs = note_attributes(action_by(event), event, resource, resource_parent) StateNote.new(attrs) end def note_html - @note_html ||= "<p dir=\"auto\">#{note_text(html: true)}</p>" + @note_html ||= Banzai::Renderer.cacheless_render_field(self, :note, { group: group, project: project }) end private def note_text(html: false) - event.state + if event.state == 'closed' + if event.close_after_error_tracking_resolve + return 'resolved the corresponding error and closed the issue.' + end + + if event.close_auto_resolve_prometheus_alert + return 'automatically closed this issue because the alert resolved.' + end + end + + body = event.state.dup + body << " via #{event_source.gfm_reference(project)}" if event_source + body + end + + def event_source + strong_memoize(:event_source) do + if event.source_commit + project&.commit(event.source_commit) + else + event.source_merge_request + end + end + end + + def self.action_by(event) + event.state == 'reopened' ? 'opened' : event.state end end diff --git a/app/models/suggestion.rb b/app/models/suggestion.rb index 96ffec90c00..94f3a140098 100644 --- a/app/models/suggestion.rb +++ b/app/models/suggestion.rb @@ -38,11 +38,18 @@ class Suggestion < ApplicationRecord end def appliable?(cached: true) - !applied? && - noteable.opened? && - !outdated?(cached: cached) && - different_content? && - note.active? + inapplicable_reason(cached: cached).nil? + end + + def inapplicable_reason(cached: true) + strong_memoize("inapplicable_reason_#{cached}") do + next :applied if applied? + next :merge_request_merged if noteable.merged? + next :merge_request_closed if noteable.closed? + next :source_branch_deleted unless noteable.source_branch_exists? + next :outdated if outdated?(cached: cached) || !note.active? + next :same_content unless different_content? + end end # Overwrites outdated column @@ -53,6 +60,10 @@ class Suggestion < ApplicationRecord from_content != fetch_from_content end + def single_line? + lines_above.zero? && lines_below.zero? + end + def target_line position.new_line end diff --git a/app/models/synthetic_note.rb b/app/models/synthetic_note.rb index 3017140f871..dea7165af9f 100644 --- a/app/models/synthetic_note.rb +++ b/app/models/synthetic_note.rb @@ -3,20 +3,18 @@ class SyntheticNote < Note attr_accessor :resource_parent, :event - self.abstract_class = true - def self.note_attributes(action, event, resource, resource_parent) resource ||= event.resource attrs = { - system: true, - author: event.user, - created_at: event.created_at, - discussion_id: event.discussion_id, - noteable: resource, - event: event, - system_note_metadata: ::SystemNoteMetadata.new(action: action), - resource_parent: resource_parent + system: true, + author: event.user, + created_at: event.created_at, + discussion_id: event.discussion_id, + noteable: resource, + event: event, + system_note_metadata: ::SystemNoteMetadata.new(action: action), + resource_parent: resource_parent } if resource_parent.is_a?(Project) diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 4e14bb4e92c..b6ba96c768e 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -18,7 +18,8 @@ class SystemNoteMetadata < ApplicationRecord designs_added designs_modified designs_removed designs_discussion_added title time_tracking branch milestone discussion task moved opened closed merged duplicate locked unlocked outdated - tag due_date pinned_embed cherry_pick health_status + tag due_date pinned_embed cherry_pick health_status approved unapproved + status alert_issue_added ].freeze validates :note, presence: true diff --git a/app/models/todo.rb b/app/models/todo.rb index 102f36a991e..f973c1ff1d4 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -7,15 +7,16 @@ class Todo < ApplicationRecord # Time to wait for todos being removed when not visible for user anymore. # Prevents TODOs being removed by mistake, for example, removing access from a user # and giving it back again. - WAIT_FOR_DELETE = 1.hour + WAIT_FOR_DELETE = 1.hour - ASSIGNED = 1 - MENTIONED = 2 - BUILD_FAILED = 3 - MARKED = 4 - APPROVAL_REQUIRED = 5 # This is an EE-only feature - UNMERGEABLE = 6 - DIRECTLY_ADDRESSED = 7 + ASSIGNED = 1 + MENTIONED = 2 + BUILD_FAILED = 3 + MARKED = 4 + APPROVAL_REQUIRED = 5 # This is an EE-only feature + UNMERGEABLE = 6 + DIRECTLY_ADDRESSED = 7 + MERGE_TRAIN_REMOVED = 8 # This is an EE-only feature ACTION_NAMES = { ASSIGNED => :assigned, @@ -24,7 +25,8 @@ class Todo < ApplicationRecord MARKED => :marked, APPROVAL_REQUIRED => :approval_required, UNMERGEABLE => :unmergeable, - DIRECTLY_ADDRESSED => :directly_addressed + DIRECTLY_ADDRESSED => :directly_addressed, + MERGE_TRAIN_REMOVED => :merge_train_removed }.freeze belongs_to :author, class_name: "User" @@ -165,6 +167,10 @@ class Todo < ApplicationRecord action == ASSIGNED end + def merge_train_removed? + action == MERGE_TRAIN_REMOVED + end + def done? state == 'done' end diff --git a/app/models/user.rb b/app/models/user.rb index 431a5b3a5b7..643b759e6f4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -69,7 +69,7 @@ class User < ApplicationRecord MINIMUM_INACTIVE_DAYS = 180 - ignore_column :ghost, remove_with: '13.2', remove_after: '2020-06-22' + ignore_column :bio, remove_with: '13.4', remove_after: '2020-09-22' # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour @@ -163,9 +163,10 @@ class User < ApplicationRecord has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id # rubocop:disable Cop/ActiveRecordDependent - has_many :issue_assignees + has_many :issue_assignees, inverse_of: :assignee + has_many :merge_request_assignees, inverse_of: :assignee has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue - has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent + has_many :assigned_merge_requests, class_name: "MergeRequest", through: :merge_request_assignees, source: :merge_request has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'UserCallout' @@ -194,7 +195,6 @@ class User < ApplicationRecord validates :notification_email, devise_email: true, if: ->(user) { user.notification_email != user.email } validates :public_email, presence: true, uniqueness: true, devise_email: true, allow_blank: true validates :commit_email, devise_email: true, allow_nil: true, if: ->(user) { user.commit_email != user.email } - validates :bio, length: { maximum: 255 }, allow_blank: true validates :projects_limit, presence: true, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: Gitlab::Database::MAX_INT_VALUE } @@ -229,7 +229,6 @@ class User < ApplicationRecord before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? } before_validation :ensure_namespace_correct before_save :ensure_namespace_correct # in case validation is skipped - before_save :ensure_bio_is_assigned_to_user_details, if: :bio_changed? after_validation :set_username_errors after_update :username_changed_hook, if: :saved_change_to_username? after_destroy :post_destroy_hook @@ -272,6 +271,7 @@ class User < ApplicationRecord :time_display_relative, :time_display_relative=, :time_format_in_24h, :time_format_in_24h=, :show_whitespace_in_diffs, :show_whitespace_in_diffs=, + :view_diffs_file_by_file, :view_diffs_file_by_file=, :tab_width, :tab_width=, :sourcegraph_enabled, :sourcegraph_enabled=, :setup_for_company, :setup_for_company=, @@ -281,6 +281,7 @@ class User < ApplicationRecord delegate :path, to: :namespace, allow_nil: true, prefix: true delegate :job_title, :job_title=, to: :user_detail, allow_nil: true + delegate :bio, :bio=, :bio_html, to: :user_detail, allow_nil: true accepts_nested_attributes_for :user_preference, update_only: true accepts_nested_attributes_for :user_detail, update_only: true @@ -619,11 +620,12 @@ class User < ApplicationRecord # Pattern used to extract `@user` user references from text def reference_pattern - %r{ - (?<!\w) - #{Regexp.escape(reference_prefix)} - (?<user>#{Gitlab::PathRegex::FULL_NAMESPACE_FORMAT_REGEX}) - }x + @reference_pattern ||= + %r{ + (?<!\w) + #{Regexp.escape(reference_prefix)} + (?<user>#{Gitlab::PathRegex::FULL_NAMESPACE_FORMAT_REGEX}) + }x end # Return (create if necessary) the ghost user. The ghost user @@ -642,6 +644,7 @@ class User < ApplicationRecord unique_internal(where(user_type: :alert_bot), 'alert-bot', email_pattern) do |u| u.bio = 'The GitLab alert bot' u.name = 'GitLab Alert Bot' + u.avatar = bot_avatar(image: 'alert-bot.png') end end @@ -655,6 +658,16 @@ class User < ApplicationRecord end end + def support_bot + email_pattern = "support%s@#{Settings.gitlab.host}" + + unique_internal(where(user_type: :support_bot), 'support-bot', email_pattern) do |u| + u.bio = 'The GitLab support bot used for Service Desk' + u.name = 'GitLab Support Bot' + u.avatar = bot_avatar(image: 'support-bot.png') + end + end + # Return true if there is only single non-internal user in the deployment, # ghost user is ignored. def single_user? @@ -1257,17 +1270,11 @@ class User < ApplicationRecord namespace.path = username if username_changed? namespace.name = name if name_changed? else - build_namespace(path: username, name: name) + namespace = build_namespace(path: username, name: name) + namespace.build_namespace_settings end end - # Temporary, will be removed when bio is fully migrated - def ensure_bio_is_assigned_to_user_details - return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true) - - user_detail.bio = bio.to_s[0...255] # bio can be NULL in users, but cannot be NULL in user_details - end - def set_username_errors namespace_path_errors = self.errors.delete(:"namespace.path") self.errors[:username].concat(namespace_path_errors) if namespace_path_errors @@ -1692,6 +1699,10 @@ class User < ApplicationRecord impersonator.present? end + def created_recently? + created_at > Devise.confirm_within.ago + end + protected # override, from Devise::Validatable diff --git a/app/models/user_callout_enums.rb b/app/models/user_callout_enums.rb index 0a3f597ae27..226c8cd9ab5 100644 --- a/app/models/user_callout_enums.rb +++ b/app/models/user_callout_enums.rb @@ -17,7 +17,8 @@ module UserCalloutEnums suggest_popover_dismissed: 9, tabs_position_highlight: 10, webhooks_moved: 13, - admin_integrations_moved: 15 + admin_integrations_moved: 15, + personal_access_token_expiry: 21 # EE-only } end end diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index 5dc74421705..9674f9a41da 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -1,7 +1,33 @@ # frozen_string_literal: true class UserDetail < ApplicationRecord + extend ::Gitlab::Utils::Override + include CacheMarkdownField + belongs_to :user validates :job_title, length: { maximum: 200 } + validates :bio, length: { maximum: 255 }, allow_blank: true + + before_save :prevent_nil_bio + + cache_markdown_field :bio + + def bio_html + read_attribute(:bio_html) || bio + end + + # For backward compatibility. + # Older migrations (and their tests) reference the `User.migration_bot` where the `bio` attribute is set. + # Here we disable writing the markdown cache when the `bio_html` column does not exists. + override :invalidated_markdown_cache? + def invalidated_markdown_cache? + self.class.column_names.include?('bio_html') && super + end + + private + + def prevent_nil_bio + self.bio = '' if bio_changed? && bio.nil? + end end diff --git a/app/models/webauthn_registration.rb b/app/models/webauthn_registration.rb new file mode 100644 index 00000000000..76f8faa11c7 --- /dev/null +++ b/app/models/webauthn_registration.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# Registration information for WebAuthn credentials + +class WebauthnRegistration < ApplicationRecord + belongs_to :user + + validates :credential_xid, :public_key, :name, :counter, presence: true + validates :counter, + numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2**32 - 1 } +end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 9e4e2f68d38..3dc90edb331 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -301,6 +301,10 @@ class WikiPage version&.commit&.committed_date end + def diffs(diff_options = {}) + Gitlab::Diff::FileCollection::WikiPage.new(self, diff_options: diff_options) + end + private def serialize_front_matter(hash) |