diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 08:27:35 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 08:27:35 +0000 |
| commit | 7e9c479f7de77702622631cff2628a9c8dcbc627 (patch) | |
| tree | c8f718a08e110ad7e1894510980d2155a6549197 /app/models | |
| parent | e852b0ae16db4052c1c567d9efa4facc81146e88 (diff) | |
| download | gitlab-ce-0bddc398e06691ecd2db73d0c570a122a6585fe8.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42
Diffstat (limited to 'app/models')
125 files changed, 1309 insertions, 445 deletions
diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb index 61cc15a522e..7ce7f40b6a8 100644 --- a/app/models/alert_management/alert.rb +++ b/app/models/alert_management/alert.rb @@ -34,7 +34,7 @@ module AlertManagement has_many :ordered_notes, -> { fresh }, as: :noteable, class_name: 'Note' has_many :user_mentions, class_name: 'AlertManagement::AlertUserMention', foreign_key: :alert_management_alert_id - has_internal_id :iid, scope: :project, init: ->(s) { s.project.alert_management_alerts.maximum(:iid) } + has_internal_id :iid, scope: :project sha_attribute :fingerprint diff --git a/app/models/alert_management/http_integration.rb b/app/models/alert_management/http_integration.rb index 7f954e1d384..ae5170867c3 100644 --- a/app/models/alert_management/http_integration.rb +++ b/app/models/alert_management/http_integration.rb @@ -2,6 +2,10 @@ module AlertManagement class HttpIntegration < ApplicationRecord + include ::Gitlab::Routing + LEGACY_IDENTIFIER = 'legacy' + DEFAULT_NAME_SLUG = 'http-endpoint' + belongs_to :project, inverse_of: :alert_management_http_integrations attr_encrypted :token, @@ -9,19 +13,45 @@ module AlertManagement key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-gcm' + default_value_for(:endpoint_identifier, allows_nil: false) { SecureRandom.hex(8) } + default_value_for(:token) { generate_token } + validates :project, presence: true validates :active, inclusion: { in: [true, false] } - - validates :token, presence: true + validates :token, presence: true, format: { with: /\A\h{32}\z/ } validates :name, presence: true, length: { maximum: 255 } - validates :endpoint_identifier, presence: true, length: { maximum: 255 } + validates :endpoint_identifier, presence: true, length: { maximum: 255 }, format: { with: /\A[A-Za-z0-9]+\z/ } validates :endpoint_identifier, uniqueness: { scope: [:project_id, :active] }, if: :active? before_validation :prevent_token_assignment + before_validation :prevent_endpoint_identifier_assignment before_validation :ensure_token + scope :for_endpoint_identifier, -> (endpoint_identifier) { where(endpoint_identifier: endpoint_identifier) } + scope :active, -> { where(active: true) } + scope :ordered_by_id, -> { order(:id) } + + def url + return project_alerts_notify_url(project, format: :json) if legacy? + + project_alert_http_integration_url(project, name_slug, endpoint_identifier, format: :json) + end + private + def self.generate_token + SecureRandom.hex + end + + def name_slug + (name && Gitlab::Utils.slugify(name)) || DEFAULT_NAME_SLUG + end + + def legacy? + endpoint_identifier == LEGACY_IDENTIFIER + end + + # Blank token assignment triggers token reset def prevent_token_assignment if token.present? && token_changed? self.token = nil @@ -31,11 +61,13 @@ module AlertManagement end def ensure_token - self.token = generate_token if token.blank? + self.token = self.class.generate_token if token.blank? end - def generate_token - SecureRandom.hex + def prevent_endpoint_identifier_assignment + if endpoint_identifier_changed? && endpoint_identifier_was.present? + self.endpoint_identifier = endpoint_identifier_was + end end end end diff --git a/app/models/analytics/devops_adoption.rb b/app/models/analytics/devops_adoption.rb new file mode 100644 index 00000000000..ed5a5b16a6e --- /dev/null +++ b/app/models/analytics/devops_adoption.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module Analytics::DevopsAdoption + def self.table_name_prefix + 'analytics_devops_adoption_' + end +end diff --git a/app/models/analytics/devops_adoption/segment.rb b/app/models/analytics/devops_adoption/segment.rb new file mode 100644 index 00000000000..71d4a312627 --- /dev/null +++ b/app/models/analytics/devops_adoption/segment.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class Analytics::DevopsAdoption::Segment < ApplicationRecord + ALLOWED_SEGMENT_COUNT = 20 + + has_many :segment_selections + has_many :groups, through: :segment_selections + + validates :name, presence: true, uniqueness: true, length: { maximum: 255 } + validate :validate_segment_count + + accepts_nested_attributes_for :segment_selections, allow_destroy: true + + scope :ordered_by_name, -> { order(:name) } + scope :with_groups, -> { preload(:groups) } + + private + + def validate_segment_count + if self.class.count >= ALLOWED_SEGMENT_COUNT + errors.add(:name, s_('DevopsAdoptionSegment|The maximum number of segments has been reached')) + end + end +end diff --git a/app/models/analytics/devops_adoption/segment_selection.rb b/app/models/analytics/devops_adoption/segment_selection.rb new file mode 100644 index 00000000000..6b70c13a773 --- /dev/null +++ b/app/models/analytics/devops_adoption/segment_selection.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class Analytics::DevopsAdoption::SegmentSelection < ApplicationRecord + ALLOWED_SELECTIONS_PER_SEGMENT = 20 + + belongs_to :segment + belongs_to :project + belongs_to :group + + validates :segment, presence: true + validates :project, presence: { unless: :group } + validates :project_id, uniqueness: { scope: :segment_id, if: :project } + validates :group, presence: { unless: :project } + validates :group_id, uniqueness: { scope: :segment_id, if: :group } + + validate :exclusive_project_or_group + validate :validate_selection_count + + private + + def exclusive_project_or_group + if project.present? && group.present? + errors.add(:group, s_('DevopsAdoptionSegmentSelection|The selection cannot be configured for a project and for a group at the same time')) + end + end + + def validate_selection_count + return unless segment + + selection_count_for_segment = self.class.where(segment: segment).count + + if selection_count_for_segment >= ALLOWED_SELECTIONS_PER_SEGMENT + errors.add(:segment, s_('DevopsAdoptionSegmentSelection|The maximum number of selections has been reached')) + end + end +end diff --git a/app/models/analytics/instance_statistics/measurement.rb b/app/models/analytics/instance_statistics/measurement.rb index 76cc1111e90..c8b76e005ef 100644 --- a/app/models/analytics/instance_statistics/measurement.rb +++ b/app/models/analytics/instance_statistics/measurement.rb @@ -15,35 +15,47 @@ module Analytics pipelines_succeeded: 7, pipelines_failed: 8, pipelines_canceled: 9, - pipelines_skipped: 10 + pipelines_skipped: 10, + billable_users: 11 } - IDENTIFIER_QUERY_MAPPING = { - identifiers[:projects] => -> { Project }, - identifiers[:users] => -> { User }, - identifiers[:issues] => -> { Issue }, - identifiers[:merge_requests] => -> { MergeRequest }, - identifiers[:groups] => -> { Group }, - identifiers[:pipelines] => -> { Ci::Pipeline }, - identifiers[:pipelines_succeeded] => -> { Ci::Pipeline.success }, - identifiers[:pipelines_failed] => -> { Ci::Pipeline.failed }, - identifiers[:pipelines_canceled] => -> { Ci::Pipeline.canceled }, - identifiers[:pipelines_skipped] => -> { Ci::Pipeline.skipped } - }.freeze - validates :recorded_at, :identifier, :count, presence: true validates :recorded_at, uniqueness: { scope: :identifier } scope :order_by_latest, -> { order(recorded_at: :desc) } scope :with_identifier, -> (identifier) { where(identifier: identifier) } + scope :recorded_after, -> (date) { where(self.model.arel_table[:recorded_at].gteq(date)) if date.present? } + scope :recorded_before, -> (date) { where(self.model.arel_table[:recorded_at].lteq(date)) if date.present? } + + def self.identifier_query_mapping + { + identifiers[:projects] => -> { Project }, + identifiers[:users] => -> { User }, + identifiers[:issues] => -> { Issue }, + identifiers[:merge_requests] => -> { MergeRequest }, + identifiers[:groups] => -> { Group }, + identifiers[:pipelines] => -> { Ci::Pipeline }, + identifiers[:pipelines_succeeded] => -> { Ci::Pipeline.success }, + identifiers[:pipelines_failed] => -> { Ci::Pipeline.failed }, + identifiers[:pipelines_canceled] => -> { Ci::Pipeline.canceled }, + identifiers[:pipelines_skipped] => -> { Ci::Pipeline.skipped } + } + end + + # Customized min and max calculation, in some cases using the original scope is too slow. + def self.identifier_min_max_queries + {} + end def self.measurement_identifier_values - if Feature.enabled?(:store_ci_pipeline_counts_by_status, default_enabled: true) - identifiers.values - else - identifiers.values - EXPERIMENTAL_IDENTIFIERS.map { |identifier| identifiers[identifier] } - end + identifiers.values + end + + def self.find_latest_or_fallback(identifier) + with_identifier(identifier).order_by_latest.first || identifier_query_mapping[identifiers[identifier]].call end end end end + +Analytics::InstanceStatistics::Measurement.prepend_if_ee('EE::Analytics::InstanceStatistics::Measurement') diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 3542bb90dc0..71235ed1002 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -48,6 +48,8 @@ class ApplicationRecord < ActiveRecord::Base def self.safe_find_or_create_by!(*args, &block) safe_find_or_create_by(*args, &block).tap do |record| + raise ActiveRecord::RecordNotFound unless record.present? + record.validate! unless record.persisted? end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index d034630a085..7bfa5fb4cb8 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -8,8 +8,6 @@ class ApplicationSetting < ApplicationRecord include IgnorableColumns ignore_column :namespace_storage_size_limit, remove_with: '13.5', remove_after: '2020-09-22' - ignore_column :instance_statistics_visibility_private, remove_with: '13.6', remove_after: '2020-10-22' - ignore_column :snowplow_iglu_registry_url, remove_with: '13.6', remove_after: '2020-11-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ @@ -42,8 +40,8 @@ class ApplicationSetting < ApplicationRecord serialize :restricted_visibility_levels # rubocop:disable Cop/ActiveRecordSerialize serialize :import_sources # rubocop:disable Cop/ActiveRecordSerialize serialize :disabled_oauth_sign_in_sources, Array # rubocop:disable Cop/ActiveRecordSerialize - serialize :domain_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize - serialize :domain_blacklist, Array # rubocop:disable Cop/ActiveRecordSerialize + serialize :domain_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize + serialize :domain_denylist, Array # rubocop:disable Cop/ActiveRecordSerialize serialize :repository_storages # rubocop:disable Cop/ActiveRecordSerialize serialize :asset_proxy_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize @@ -186,9 +184,9 @@ class ApplicationSetting < ApplicationRecord validates :enabled_git_access_protocol, inclusion: { in: %w(ssh http), allow_blank: true } - validates :domain_blacklist, - presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' }, - if: :domain_blacklist_enabled? + validates :domain_denylist, + presence: { message: 'Domain denylist cannot be empty if denylist is enabled.' }, + if: :domain_denylist_enabled? validates :housekeeping_incremental_repack_period, presence: true, @@ -294,6 +292,9 @@ class ApplicationSetting < ApplicationRecord validates :container_registry_delete_tags_service_timeout, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :container_registry_expiration_policies_worker_capacity, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end @@ -385,6 +386,9 @@ class ApplicationSetting < ApplicationRecord validates :raw_blob_request_limit, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :ci_jwt_signing_key, + rsa_key: true, allow_nil: true + attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, @@ -410,6 +414,9 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :recaptcha_site_key, encryption_options_base_truncated_aes_256_gcm attr_encrypted :slack_app_secret, encryption_options_base_truncated_aes_256_gcm attr_encrypted :slack_app_verification_token, encryption_options_base_truncated_aes_256_gcm + attr_encrypted :ci_jwt_signing_key, encryption_options_base_truncated_aes_256_gcm + attr_encrypted :secret_detection_token_revocation_token, encryption_options_base_truncated_aes_256_gcm + attr_encrypted :cloud_license_auth_token, encryption_options_base_truncated_aes_256_gcm before_validation :ensure_uuid! diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 8a7bd5a7ad9..5c7abbccd63 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -60,7 +60,7 @@ module ApplicationSettingImplementation diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, disabled_oauth_sign_in_sources: [], dns_rebinding_protection_enabled: true, - domain_whitelist: Settings.gitlab['domain_whitelist'], + domain_allowlist: Settings.gitlab['domain_allowlist'], dsa_key_restriction: 0, ecdsa_key_restriction: 0, ed25519_key_restriction: 0, @@ -120,7 +120,7 @@ module ApplicationSettingImplementation repository_checks_enabled: true, repository_storages_weighted: { default: 100 }, repository_storages: ['default'], - require_admin_approval_after_user_signup: false, + require_admin_approval_after_user_signup: true, require_two_factor_authentication: false, restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], rsa_key_restriction: 0, @@ -167,7 +167,8 @@ module ApplicationSettingImplementation user_default_internal_regex: nil, user_show_add_ssh_key_message: true, wiki_page_max_content_bytes: 50.megabytes, - container_registry_delete_tags_service_timeout: 100 + container_registry_delete_tags_service_timeout: 250, + container_registry_expiration_policies_worker_capacity: 0 } end @@ -201,38 +202,38 @@ module ApplicationSettingImplementation super(sources) end - def domain_whitelist_raw - array_to_string(self.domain_whitelist) + def domain_allowlist_raw + array_to_string(self.domain_allowlist) end - def domain_blacklist_raw - array_to_string(self.domain_blacklist) + def domain_denylist_raw + array_to_string(self.domain_denylist) end - def domain_whitelist_raw=(values) - self.domain_whitelist = strings_to_array(values) + def domain_allowlist_raw=(values) + self.domain_allowlist = strings_to_array(values) end - def domain_blacklist_raw=(values) - self.domain_blacklist = strings_to_array(values) + def domain_denylist_raw=(values) + self.domain_denylist = strings_to_array(values) end - def domain_blacklist_file=(file) - self.domain_blacklist_raw = file.read + def domain_denylist_file=(file) + self.domain_denylist_raw = file.read end - def outbound_local_requests_whitelist_raw + def outbound_local_requests_allowlist_raw array_to_string(self.outbound_local_requests_whitelist) end - def outbound_local_requests_whitelist_raw=(values) - clear_memoization(:outbound_local_requests_whitelist_arrays) + def outbound_local_requests_allowlist_raw=(values) + clear_memoization(:outbound_local_requests_allowlist_arrays) self.outbound_local_requests_whitelist = strings_to_array(values) end def add_to_outbound_local_requests_whitelist(values_array) - clear_memoization(:outbound_local_requests_whitelist_arrays) + clear_memoization(:outbound_local_requests_allowlist_arrays) self.outbound_local_requests_whitelist ||= [] self.outbound_local_requests_whitelist += values_array @@ -244,13 +245,13 @@ module ApplicationSettingImplementation # application_setting.outbound_local_requests_whitelist array into 2 arrays; # an array of IPAddr objects (`[IPAddr.new('127.0.0.1')]`), and an array of # domain strings (`['www.example.com']`). - def outbound_local_requests_whitelist_arrays - strong_memoize(:outbound_local_requests_whitelist_arrays) do + def outbound_local_requests_allowlist_arrays + strong_memoize(:outbound_local_requests_allowlist_arrays) do next [[], []] unless self.outbound_local_requests_whitelist - ip_whitelist, domain_whitelist = separate_whitelists(self.outbound_local_requests_whitelist) + ip_allowlist, domain_allowlist = separate_allowlists(self.outbound_local_requests_whitelist) - [ip_whitelist, domain_whitelist] + [ip_allowlist, domain_allowlist] end end @@ -395,19 +396,19 @@ module ApplicationSettingImplementation private - def separate_whitelists(string_array) - string_array.reduce([[], []]) do |(ip_whitelist, domain_whitelist), string| + def separate_allowlists(string_array) + string_array.reduce([[], []]) do |(ip_allowlist, domain_allowlist), string| address, port = parse_addr_and_port(string) ip_obj = Gitlab::Utils.string_to_ip_object(address) if ip_obj - ip_whitelist << Gitlab::UrlBlockers::IpWhitelistEntry.new(ip_obj, port: port) + ip_allowlist << Gitlab::UrlBlockers::IpAllowlistEntry.new(ip_obj, port: port) else - domain_whitelist << Gitlab::UrlBlockers::DomainWhitelistEntry.new(address, port: port) + domain_allowlist << Gitlab::UrlBlockers::DomainAllowlistEntry.new(address, port: port) end - [ip_whitelist, domain_whitelist] + [ip_allowlist, domain_allowlist] end end diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 34f03e769a0..55e8a5d4535 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -2,7 +2,6 @@ class AuditEvent < ApplicationRecord include CreatedAtFilterable - include IgnorableColumns include BulkInsertSafe include EachBatch @@ -14,8 +13,6 @@ class AuditEvent < ApplicationRecord :target_id ].freeze - ignore_column :type, remove_with: '13.6', remove_after: '2020-11-22' - serialize :details, Hash # rubocop:disable Cop/ActiveRecordSerialize belongs_to :user, foreign_key: :author_id @@ -37,14 +34,6 @@ class AuditEvent < ApplicationRecord # https://gitlab.com/groups/gitlab-org/-/epics/2765 after_validation :parallel_persist - # Note: After loading records, do not attempt to type cast objects it finds. - # We are in the process of deprecating STI (i.e. SecurityEvent) out of AuditEvent. - # - # https://gitlab.com/gitlab-org/gitlab/-/issues/216845 - def self.inheritance_column - :_type_disabled - end - def self.order_by(method) case method.to_s when 'created_asc' diff --git a/app/models/authentication_event.rb b/app/models/authentication_event.rb index ac6e08caf50..9d191e6ae4d 100644 --- a/app/models/authentication_event.rb +++ b/app/models/authentication_event.rb @@ -3,6 +3,12 @@ class AuthenticationEvent < ApplicationRecord include UsageStatistics + TWO_FACTOR = 'two-factor'.freeze + TWO_FACTOR_U2F = 'two-factor-via-u2f-device'.freeze + TWO_FACTOR_WEBAUTHN = 'two-factor-via-webauthn-device'.freeze + STANDARD = 'standard'.freeze + STATIC_PROVIDERS = [TWO_FACTOR, TWO_FACTOR_U2F, TWO_FACTOR_WEBAUTHN, STANDARD].freeze + belongs_to :user, optional: true validates :provider, :user_name, :result, presence: true @@ -17,6 +23,6 @@ class AuthenticationEvent < ApplicationRecord scope :ldap, -> { where('provider LIKE ?', 'ldap%')} def self.providers - distinct.pluck(:provider) + STATIC_PROVIDERS | Devise.omniauth_providers.map(&:to_s) end end diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 856f86201ec..a8325e98095 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -103,6 +103,7 @@ class BroadcastMessage < ApplicationRecord end def matches_current_path(current_path) + return false if current_path.blank? && target_path.present? return true if current_path.blank? || target_path.blank? escaped = Regexp.escape(target_path).gsub('\\*', '.*') diff --git a/app/models/bulk_import.rb b/app/models/bulk_import.rb index cabff86a9f9..5d646313423 100644 --- a/app/models/bulk_import.rb +++ b/app/models/bulk_import.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# The BulkImport model links all models required for a bulk import of groups and +# projects to a GitLab instance. It associates the import with the responsible +# user. class BulkImport < ApplicationRecord belongs_to :user, optional: false @@ -12,5 +15,20 @@ class BulkImport < ApplicationRecord state_machine :status, initial: :created do state :created, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :failed, value: -1 + + event :start do + transition created: :started + end + + event :finish do + transition started: :finished + end + + event :fail_op do + transition any => :failed + end end end diff --git a/app/models/bulk_imports/configuration.rb b/app/models/bulk_imports/configuration.rb index 8c3aff6f749..4c6f745c268 100644 --- a/app/models/bulk_imports/configuration.rb +++ b/app/models/bulk_imports/configuration.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# Stores the authentication data required to access another GitLab instance on +# behalf of a user, to import Groups and Projects directly from that instance. class BulkImports::Configuration < ApplicationRecord self.table_name = 'bulk_import_configurations' diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 2d0bba7bccc..34030e079c7 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -1,5 +1,22 @@ # frozen_string_literal: true +# The BulkImport::Entity represents a Group or Project to be imported during the +# bulk import process. An entity is nested under the parent group when it is not +# a top level group. +# +# A full bulk import entity structure might look like this, where the links are +# parents: +# +# **Before Import** **After Import** +# +# GroupEntity Group +# | | | | +# GroupEntity ProjectEntity Group Project +# | | +# ProjectEntity Project +# +# The tree structure of the entities results in the same structure for imported +# Groups and Projects. class BulkImports::Entity < ApplicationRecord self.table_name = 'bulk_import_entities' @@ -9,6 +26,10 @@ class BulkImports::Entity < ApplicationRecord belongs_to :project, optional: true belongs_to :group, foreign_key: :namespace_id, optional: true + has_many :trackers, + class_name: 'BulkImports::Tracker', + foreign_key: :bulk_import_entity_id + validates :project, absence: true, if: :group validates :group, absence: true, if: :project validates :source_type, :source_full_path, :destination_name, @@ -21,6 +42,21 @@ class BulkImports::Entity < ApplicationRecord state_machine :status, initial: :created do state :created, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :failed, value: -1 + + event :start do + transition created: :started + end + + event :finish do + transition started: :finished + end + + event :fail_op do + transition any => :failed + end end private @@ -33,11 +69,17 @@ class BulkImports::Entity < ApplicationRecord def validate_imported_entity_type if group.present? && project_entity? - errors.add(:group, s_('BulkImport|expected an associated Project but has an associated Group')) + errors.add( + :group, + s_('BulkImport|expected an associated Project but has an associated Group') + ) end if project.present? && group_entity? - errors.add(:project, s_('BulkImport|expected an associated Group but has an associated Project')) + errors.add( + :project, + s_('BulkImport|expected an associated Group but has an associated Project') + ) end end end diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb new file mode 100644 index 00000000000..02e0904e1af --- /dev/null +++ b/app/models/bulk_imports/tracker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# This model is responsible for keeping track of the requests/pagination +# happening during a Group Migration (BulkImport). +class BulkImports::Tracker < ApplicationRecord + self.table_name = 'bulk_import_trackers' + + belongs_to :entity, + class_name: 'BulkImports::Entity', + foreign_key: :bulk_import_entity_id, + optional: false + + validates :relation, + presence: true, + uniqueness: { scope: :bulk_import_entity_id } + + validates :next_page, presence: { if: :has_next_page? } +end diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 2e725e0baff..5b23cf46fdb 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -7,6 +7,7 @@ module Ci include Importable include AfterCommitQueue include Ci::HasRef + extend ::Gitlab::Utils::Override InvalidBridgeTypeError = Class.new(StandardError) InvalidTransitionError = Class.new(StandardError) @@ -203,8 +204,11 @@ module Ci end end + override :dependency_variables def dependency_variables - [] + return [] unless ::Feature.enabled?(:ci_bridge_dependency_variables, project) + + super end def target_revision_ref diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 9ff70ece947..84abd01786d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -103,6 +103,10 @@ module Ci ) end + scope :in_pipelines, ->(pipelines) do + where(pipeline: pipelines) + end + scope :with_existing_job_artifacts, ->(query) do where('EXISTS (?)', ::Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').merge(query)) end @@ -571,14 +575,6 @@ module Ci end end - def dependency_variables - return [] if all_dependencies.empty? - - Gitlab::Ci::Variables::Collection.new.concat( - Ci::JobVariable.where(job: all_dependencies).dotenv_source - ) - end - def features { trace_sections: true } end @@ -828,10 +824,6 @@ module Ci Gitlab::Ci::Build::Credentials::Factory.new(self).create! end - def all_dependencies - dependencies.all - end - def has_valid_build_dependencies? dependencies.valid? end @@ -994,12 +986,6 @@ module Ci end end - def dependencies - strong_memoize(:dependencies) do - Ci::BuildDependencies.new(self) - end - end - def build_data @build_data ||= Gitlab::DataBuilder::Build.build(self) end @@ -1059,7 +1045,7 @@ module Ci jwt = Gitlab::Ci::Jwt.for_build(self) variables.append(key: 'CI_JOB_JWT', value: jwt, public: false, masked: true) - rescue OpenSSL::PKey::RSAError => e + rescue OpenSSL::PKey::RSAError, Gitlab::Ci::Jwt::NoSigningKeyError => e Gitlab::ErrorTracking.track_exception(e) end end diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index cf6eb159f52..ceefb6a8b8a 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -22,20 +22,26 @@ module Ci FailedToPersistDataError = Class.new(StandardError) - # Note: The ordering of this enum is related to the precedence of persist store. + # Note: The ordering of this hash is related to the precedence of persist store. # The bottom item takes the highest precedence, and the top item takes the lowest precedence. - enum data_store: { + DATA_STORES = { redis: 1, database: 2, fog: 3 - } + }.freeze + + STORE_TYPES = DATA_STORES.keys.map do |store| + [store, "Ci::BuildTraceChunks::#{store.capitalize}".constantize] + end.to_h.freeze + + enum data_store: DATA_STORES scope :live, -> { redis } scope :persisted, -> { not_redis.order(:chunk_index) } class << self def all_stores - @all_stores ||= self.data_stores.keys + STORE_TYPES.keys end def persistable_store @@ -44,12 +50,11 @@ module Ci end def get_store_class(store) - @stores ||= {} + store = store.to_sym - # Can't memoize this because the feature flag may alter this - return fog_store_class.new if store.to_sym == :fog + raise "Unknown store type: #{store}" unless STORE_TYPES.key?(store) - @stores[store] ||= "Ci::BuildTraceChunks::#{store.capitalize}".constantize.new + STORE_TYPES[store].new end ## @@ -78,14 +83,6 @@ module Ci def metadata_attributes attribute_names - %w[raw_data] end - - def fog_store_class - if Feature.enabled?(:ci_trace_new_fog_store, default_enabled: true) - Ci::BuildTraceChunks::Fog - else - Ci::BuildTraceChunks::LegacyFog - end - end end def data @@ -108,7 +105,7 @@ module Ci raise ArgumentError, 'Offset is out of range' if offset < 0 || offset > size raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize) - in_lock(*lock_params) { unsafe_append_data!(new_data, offset) } + in_lock(lock_key, **lock_params) { unsafe_append_data!(new_data, offset) } schedule_to_persist! if full? end @@ -148,12 +145,13 @@ module Ci # We are using optimistic locking combined with Redis locking to ensure # that a chunk gets migrated properly. # - # We are catching an exception related to an exclusive lock not being - # acquired because it is creating a lot of noise, and is a result of - # duplicated workers running in parallel for the same build trace chunk. + # We are using until_executed deduplication strategy for workers, + # which should prevent duplicated workers running in parallel for the same build trace, + # and causing an exception related to an exclusive lock not being + # acquired # def persist_data! - in_lock(*lock_params) do # exclusive Redis lock is acquired first + in_lock(lock_key, **lock_params) do # exclusive Redis lock is acquired first raise FailedToPersistDataError, 'Modifed build trace chunk detected' if has_changes_to_save? self.reset.then do |chunk| # we ensure having latest lock_version @@ -162,6 +160,8 @@ module Ci end rescue FailedToObtainLockError metrics.increment_trace_operation(operation: :stalled) + + raise FailedToPersistDataError, 'Data migration failed due to a worker duplication' rescue ActiveRecord::StaleObjectError raise FailedToPersistDataError, <<~MSG Data migration race condition detected @@ -289,11 +289,16 @@ module Ci build.trace_chunks.maximum(:chunk_index).to_i end + def lock_key + "trace_write:#{build_id}:chunks:#{chunk_index}" + end + def lock_params - ["trace_write:#{build_id}:chunks:#{chunk_index}", - { ttl: WRITE_LOCK_TTL, - retries: WRITE_LOCK_RETRY, - sleep_sec: WRITE_LOCK_SLEEP }] + { + ttl: WRITE_LOCK_TTL, + retries: WRITE_LOCK_RETRY, + sleep_sec: WRITE_LOCK_SLEEP + } end def metrics diff --git a/app/models/ci/build_trace_chunks/legacy_fog.rb b/app/models/ci/build_trace_chunks/legacy_fog.rb deleted file mode 100644 index b710ed2890b..00000000000 --- a/app/models/ci/build_trace_chunks/legacy_fog.rb +++ /dev/null @@ -1,77 +0,0 @@ -# frozen_string_literal: true - -module Ci - module BuildTraceChunks - class LegacyFog - def available? - object_store.enabled - end - - def data(model) - connection.get_object(bucket_name, key(model))[:body] - rescue Excon::Error::NotFound - # If the object does not exist in the object storage, this method returns nil. - end - - def set_data(model, new_data) - connection.put_object(bucket_name, key(model), new_data) - end - - def append_data(model, new_data, offset) - if offset > 0 - truncated_data = data(model).to_s.byteslice(0, offset) - new_data = truncated_data + new_data - end - - set_data(model, new_data) - new_data.bytesize - end - - def size(model) - data(model).to_s.bytesize - end - - def delete_data(model) - delete_keys([[model.build_id, model.chunk_index]]) - end - - def keys(relation) - return [] unless available? - - relation.pluck(:build_id, :chunk_index) - end - - def delete_keys(keys) - keys.each do |key| - connection.delete_object(bucket_name, key_raw(*key)) - end - end - - private - - def key(model) - key_raw(model.build_id, model.chunk_index) - end - - def key_raw(build_id, chunk_index) - "tmp/builds/#{build_id.to_i}/chunks/#{chunk_index.to_i}.log" - end - - def bucket_name - return unless available? - - object_store.remote_directory - end - - def connection - return unless available? - - @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys) - end - - def object_store - Gitlab.config.artifacts.object_store - end - end - end -end diff --git a/app/models/ci/daily_build_group_report_result.rb b/app/models/ci/daily_build_group_report_result.rb index e6f02f2e4f3..e9f3366b939 100644 --- a/app/models/ci/daily_build_group_report_result.rb +++ b/app/models/ci/daily_build_group_report_result.rb @@ -4,6 +4,7 @@ module Ci class DailyBuildGroupReportResult < ApplicationRecord extend Gitlab::Ci::Model + REPORT_WINDOW = 90.days PARAM_TYPES = %w[coverage].freeze belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id @@ -12,13 +13,30 @@ module Ci validates :data, json_schema: { filename: "daily_build_group_report_result_data" } scope :with_included_projects, -> { includes(:project) } + scope :by_projects, -> (ids) { where(project_id: ids) } + scope :with_coverage, -> { where("(data->'coverage') IS NOT NULL") } + scope :with_default_branch, -> { where(default_branch: true) } + scope :by_date, -> (start_date) { where(date: report_window(start_date)..Date.current) } - def self.upsert_reports(data) - upsert_all(data, unique_by: :index_daily_build_group_report_results_unique_columns) if data.any? - end + store_accessor :data, :coverage + + class << self + def upsert_reports(data) + upsert_all(data, unique_by: :index_daily_build_group_report_results_unique_columns) if data.any? + end + + def recent_results(attrs, limit: nil) + where(attrs).order(date: :desc, group_name: :asc).limit(limit) + end - def self.recent_results(attrs, limit: nil) - where(attrs).order(date: :desc, group_name: :asc).limit(limit) + def report_window(start_date) + default_date = REPORT_WINDOW.ago.to_date + date = Date.parse(start_date) rescue default_date + + [date, default_date].max + end end end end + +Ci::DailyBuildGroupReportResult.prepend_if_ee('EE::Ci::DailyBuildGroupReportResult') diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 02e17afdab0..7cedd13b407 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -169,6 +169,7 @@ module Ci scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked).order(expire_at: :desc) } + scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) } scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 684b6387ab1..8707d635e03 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -42,9 +42,16 @@ module Ci belongs_to :external_pull_request belongs_to :ci_ref, class_name: 'Ci::Ref', foreign_key: :ci_ref_id, inverse_of: :pipelines - has_internal_id :iid, scope: :project, presence: false, track_if: -> { !importing? }, ensure_if: -> { !importing? }, init: ->(s) do - s&.project&.all_pipelines&.maximum(:iid) || s&.project&.all_pipelines&.count - end + has_internal_id :iid, scope: :project, presence: false, + track_if: -> { !importing? }, + ensure_if: -> { !importing? }, + init: ->(pipeline, scope) do + if pipeline + pipeline.project&.all_pipelines&.maximum(:iid) || pipeline.project&.all_pipelines&.count + elsif scope + ::Ci::Pipeline.where(**scope).maximum(:iid) + end + end has_many :stages, -> { order(position: :asc) }, inverse_of: :pipeline has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline @@ -270,6 +277,7 @@ module Ci scope :internal, -> { where(source: internal_sources) } scope :no_child, -> { where.not(source: :parent_pipeline) } scope :ci_sources, -> { where(source: Enums::Ci::Pipeline.ci_sources.values) } + scope :ci_and_parent_sources, -> { where(source: Enums::Ci::Pipeline.ci_and_parent_sources.values) } scope :for_user, -> (user) { where(user: user) } scope :for_sha, -> (sha) { where(sha: sha) } scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) } @@ -347,6 +355,14 @@ module Ci end end + def self.latest_running_for_ref(ref) + newest_first(ref: ref).running.take + end + + def self.latest_failed_for_ref(ref) + newest_first(ref: ref).failed.take + end + # Returns a Hash containing the latest pipeline for every given # commit. # @@ -926,7 +942,7 @@ module Ci def accessibility_reports Gitlab::Ci::Reports::AccessibilityReports.new.tap do |accessibility_reports| - builds.latest.with_reports(Ci::JobArtifact.accessibility_reports).each do |build| + latest_report_builds(Ci::JobArtifact.accessibility_reports).each do |build| build.collect_accessibility_reports!(accessibility_reports) end end diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index ac5785d9c91..6aaf6ac530b 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -103,5 +103,25 @@ module Ci pipeline.ensure_scheduling_type! reset end + + def dependency_variables + return [] if all_dependencies.empty? + + Gitlab::Ci::Variables::Collection.new.concat( + Ci::JobVariable.where(job: all_dependencies).dotenv_source + ) + end + + def all_dependencies + dependencies.all + end + + private + + def dependencies + strong_memoize(:dependencies) do + Ci::BuildDependencies.new(self) + end + end end end diff --git a/app/models/ci/test_case.rb b/app/models/ci/test_case.rb new file mode 100644 index 00000000000..19ecc177436 --- /dev/null +++ b/app/models/ci/test_case.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Ci + class TestCase < ApplicationRecord + extend Gitlab::Ci::Model + + validates :project, :key_hash, presence: true + + has_many :test_case_failures, class_name: 'Ci::TestCaseFailure' + + belongs_to :project + + scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) } + + class << self + def find_or_create_by_batch(project, test_case_keys) + # Insert records first. Existing ones will be skipped. + insert_all(test_case_attrs(project, test_case_keys)) + + # Find all matching records now that we are sure they all are persisted. + by_project_and_keys(project, test_case_keys) + end + + private + + def test_case_attrs(project, test_case_keys) + # NOTE: Rails 6.1 will add support for insert_all on relation so that + # we will be able to do project.test_cases.insert_all. + test_case_keys.map do |hashed_key| + { project_id: project.id, key_hash: hashed_key } + end + end + end + end +end diff --git a/app/models/ci/test_case_failure.rb b/app/models/ci/test_case_failure.rb new file mode 100644 index 00000000000..8867b954240 --- /dev/null +++ b/app/models/ci/test_case_failure.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Ci + class TestCaseFailure < ApplicationRecord + extend Gitlab::Ci::Model + + REPORT_WINDOW = 14.days + + validates :test_case, :build, :failed_at, presence: true + + belongs_to :test_case, class_name: "Ci::TestCase", foreign_key: :test_case_id + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id + + def self.recent_failures_count(project:, test_case_keys:, date_range: REPORT_WINDOW.ago..Time.current) + joins(:test_case) + .where( + ci_test_cases: { + project_id: project.id, + key_hash: test_case_keys + }, + ci_test_case_failures: { + failed_at: date_range + } + ) + .group(:key_hash) + .count('ci_test_case_failures.id') + end + end +end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index e9f1ee4e033..5c9561ffa98 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -3,7 +3,7 @@ module Clusters class AgentToken < ApplicationRecord include TokenAuthenticatable - add_authentication_token_field :token, encrypted: :required + add_authentication_token_field :token, encrypted: :required, token_generator: -> { Devise.friendly_token(50) } self.table_name = 'cluster_agent_tokens' diff --git a/app/models/clusters/applications/cert_manager.rb b/app/models/clusters/applications/cert_manager.rb index 1efa44c39c5..d32fff14590 100644 --- a/app/models/clusters/applications/cert_manager.rb +++ b/app/models/clusters/applications/cert_manager.rb @@ -30,7 +30,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: 'certmanager', repository: repository, version: VERSION, @@ -43,7 +43,7 @@ module Clusters end def uninstall_command - Gitlab::Kubernetes::Helm::DeleteCommand.new( + helm_command_module::DeleteCommand.new( name: 'certmanager', rbac: cluster.platform_kubernetes_rbac?, files: files, diff --git a/app/models/clusters/applications/crossplane.rb b/app/models/clusters/applications/crossplane.rb index 420e56c1742..2b1a86706a4 100644 --- a/app/models/clusters/applications/crossplane.rb +++ b/app/models/clusters/applications/crossplane.rb @@ -29,7 +29,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: 'crossplane', repository: repository, version: VERSION, diff --git a/app/models/clusters/applications/elastic_stack.rb b/app/models/clusters/applications/elastic_stack.rb index 77996748b81..db18a29ec84 100644 --- a/app/models/clusters/applications/elastic_stack.rb +++ b/app/models/clusters/applications/elastic_stack.rb @@ -26,7 +26,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: 'elastic-stack', version: VERSION, rbac: cluster.platform_kubernetes_rbac?, @@ -39,7 +39,7 @@ module Clusters end def uninstall_command - Gitlab::Kubernetes::Helm::DeleteCommand.new( + helm_command_module::DeleteCommand.new( name: 'elastic-stack', rbac: cluster.platform_kubernetes_rbac?, files: files, @@ -96,7 +96,7 @@ module Clusters def post_install_script [ - "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-master:9200" + "timeout 60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-master:9200" ] end @@ -116,7 +116,7 @@ module Clusters # Chart version 3.0.0 moves to our own chart at https://gitlab.com/gitlab-org/charts/elastic-stack # and is not compatible with pre-existing resources. We first remove them. [ - Gitlab::Kubernetes::Helm::DeleteCommand.new( + helm_command_module::DeleteCommand.new( name: 'elastic-stack', rbac: cluster.platform_kubernetes_rbac?, files: files diff --git a/app/models/clusters/applications/fluentd.rb b/app/models/clusters/applications/fluentd.rb index c608d37be77..91aa422b859 100644 --- a/app/models/clusters/applications/fluentd.rb +++ b/app/models/clusters/applications/fluentd.rb @@ -30,7 +30,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: 'fluentd', repository: repository, version: VERSION, diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb index 4a1bcac4bb7..d1d6defb713 100644 --- a/app/models/clusters/applications/helm.rb +++ b/app/models/clusters/applications/helm.rb @@ -4,6 +4,8 @@ require 'openssl' module Clusters module Applications + # DEPRECATED: This model represents the Helm 2 Tiller server, and is no longer being actively used. + # It is being kept around for a potential cleanup of the unused Tiller server. class Helm < ApplicationRecord self.table_name = 'clusters_applications_helm' @@ -49,7 +51,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InitCommand.new( + Gitlab::Kubernetes::Helm::V2::InitCommand.new( name: name, files: files, rbac: cluster.platform_kubernetes_rbac? @@ -57,7 +59,7 @@ module Clusters end def uninstall_command - Gitlab::Kubernetes::Helm::ResetCommand.new( + Gitlab::Kubernetes::Helm::V2::ResetCommand.new( name: name, files: files, rbac: cluster.platform_kubernetes_rbac? @@ -86,19 +88,19 @@ module Clusters end def create_keys_and_certs - ca_cert = Gitlab::Kubernetes::Helm::Certificate.generate_root + ca_cert = Gitlab::Kubernetes::Helm::V2::Certificate.generate_root self.ca_key = ca_cert.key_string self.ca_cert = ca_cert.cert_string end def tiller_cert - @tiller_cert ||= ca_cert_obj.issue(expires_in: Gitlab::Kubernetes::Helm::Certificate::INFINITE_EXPIRY) + @tiller_cert ||= ca_cert_obj.issue(expires_in: Gitlab::Kubernetes::Helm::V2::Certificate::INFINITE_EXPIRY) end def ca_cert_obj return unless has_ssl? - Gitlab::Kubernetes::Helm::Certificate + Gitlab::Kubernetes::Helm::V2::Certificate .from_strings(ca_key, ca_cert) end end diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index d5412714858..36324e7f3e0 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -62,7 +62,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: name, repository: repository, version: VERSION, diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index 056ea355de6..ff907c6847f 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -39,7 +39,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: name, version: VERSION, rbac: cluster.platform_kubernetes_rbac?, diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb index 3047da12dd9..b1c3116d77c 100644 --- a/app/models/clusters/applications/knative.rb +++ b/app/models/clusters/applications/knative.rb @@ -70,7 +70,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: name, version: VERSION, rbac: cluster.platform_kubernetes_rbac?, @@ -94,7 +94,7 @@ module Clusters end def uninstall_command - Gitlab::Kubernetes::Helm::DeleteCommand.new( + helm_command_module::DeleteCommand.new( name: name, rbac: cluster.platform_kubernetes_rbac?, files: files, diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 7679296699f..55a9a0ccb81 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -67,7 +67,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: name, repository: repository, version: VERSION, @@ -79,7 +79,7 @@ module Clusters end def patch_command(values) - ::Gitlab::Kubernetes::Helm::PatchCommand.new( + helm_command_module::PatchCommand.new( name: name, repository: repository, version: version, @@ -90,7 +90,7 @@ module Clusters end def uninstall_command - Gitlab::Kubernetes::Helm::DeleteCommand.new( + helm_command_module::DeleteCommand.new( name: name, rbac: cluster.platform_kubernetes_rbac?, files: files, diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index d07ea7b71dc..03f4caccccd 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.21.1' + VERSION = '0.22.0' self.table_name = 'clusters_applications_runners' @@ -30,7 +30,7 @@ module Clusters end def install_command - Gitlab::Kubernetes::Helm::InstallCommand.new( + helm_command_module::InstallCommand.new( name: name, version: VERSION, rbac: cluster.platform_kubernetes_rbac?, diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index b94ec3c6dea..3cf5542ae76 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -79,6 +79,9 @@ module Clusters validates :cluster_type, presence: true validates :domain, allow_blank: true, hostname: { allow_numeric_hostname: true } validates :namespace_per_environment, inclusion: { in: [true, false] } + validates :helm_major_version, inclusion: { in: [2, 3] } + + default_value_for :helm_major_version, 3 validate :restrict_modification, on: :update validate :no_groups, unless: :group_type? diff --git a/app/models/clusters/concerns/application_core.rb b/app/models/clusters/concerns/application_core.rb index 760576ea1eb..b82b1887308 100644 --- a/app/models/clusters/concerns/application_core.rb +++ b/app/models/clusters/concerns/application_core.rb @@ -12,6 +12,17 @@ module Clusters after_initialize :set_initial_status + def helm_command_module + case cluster.helm_major_version + when 3 + Gitlab::Kubernetes::Helm::V3 + when 2 + Gitlab::Kubernetes::Helm::V2 + else + raise "Invalid Helm major version" + end + end + def set_initial_status return unless not_installable? diff --git a/app/models/clusters/concerns/application_data.rb b/app/models/clusters/concerns/application_data.rb index 22e597e9747..00aeb7669ad 100644 --- a/app/models/clusters/concerns/application_data.rb +++ b/app/models/clusters/concerns/application_data.rb @@ -4,7 +4,7 @@ module Clusters module Concerns module ApplicationData def uninstall_command - Gitlab::Kubernetes::Helm::DeleteCommand.new( + helm_command_module::DeleteCommand.new( name: name, rbac: cluster.platform_kubernetes_rbac?, files: files diff --git a/app/models/clusters/providers/aws.rb b/app/models/clusters/providers/aws.rb index 35e8b751b3d..bfd01775620 100644 --- a/app/models/clusters/providers/aws.rb +++ b/app/models/clusters/providers/aws.rb @@ -5,9 +5,6 @@ module Clusters class Aws < ApplicationRecord include Gitlab::Utils::StrongMemoize include Clusters::Concerns::ProviderStatus - include IgnorableColumns - - ignore_column :created_by_user_id, remove_with: '13.4', remove_after: '2020-08-22' self.table_name = 'cluster_providers_aws' diff --git a/app/models/commit.rb b/app/models/commit.rb index 83400c9e533..80dd02981c1 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -335,7 +335,11 @@ class Commit strong_memoize(:raw_signature_type) do next unless @raw.instance_of?(Gitlab::Git::Commit) - @raw.raw_commit.signature_type if defined? @raw.raw_commit.signature_type + if raw_commit_from_rugged? && gpg_commit.signature_text.present? + :PGP + elsif defined? @raw.raw_commit.signature_type + @raw.raw_commit.signature_type + end end end @@ -347,7 +351,7 @@ class Commit strong_memoize(:signature) do case signature_type when :PGP - Gitlab::Gpg::Commit.new(self).signature + gpg_commit.signature when :X509 Gitlab::X509::Commit.new(self).signature else @@ -356,6 +360,14 @@ class Commit end end + def raw_commit_from_rugged? + @raw.raw_commit.is_a?(Rugged::Commit) + end + + def gpg_commit + @gpg_commit ||= Gitlab::Gpg::Commit.new(self) + end + def revert_branch_name "revert-#{short_id}" end diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 4498e08d754..ee9c2501bfc 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -205,13 +205,8 @@ class CommitStatus < ApplicationRecord # 'rspec:linux: 1/10' => 'rspec:linux' common_name = name.to_s.gsub(%r{\d+[\s:\/\\]+\d+\s*}, '') - if ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? - # 'rspec:linux: [aws, max memory]' => 'rspec:linux', 'rspec:linux: [aws]' => 'rspec:linux' - common_name.gsub!(%r{: \[.*\]\s*\z}, '') - else - # 'rspec:linux: [aws, max memory]' => 'rspec:linux', 'rspec:linux: [aws]' => 'rspec:linux: [aws]' - common_name.gsub!(%r{: \[.*, .*\]\s*\z}, '') - end + # 'rspec:linux: [aws, max memory]' => 'rspec:linux', 'rspec:linux: [aws]' => 'rspec:linux' + common_name.gsub!(%r{: \[.*\]\s*\z}, '') common_name.strip! common_name diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 4a632e8cd0c..baa99fa5a7f 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -27,16 +27,42 @@ module AtomicInternalId extend ActiveSupport::Concern class_methods do - def has_internal_id(column, scope:, init:, ensure_if: nil, track_if: nil, presence: true, backfill: false) # rubocop:disable Naming/PredicateName - # We require init here to retain the ability to recalculate in the absence of a - # InternalId record (we may delete records in `internal_ids` for example). - raise "has_internal_id requires a init block, none given." unless init + def has_internal_id( # rubocop:disable Naming/PredicateName + column, scope:, init: :not_given, ensure_if: nil, track_if: nil, + presence: true, backfill: false, hook_names: :create) + raise "has_internal_id init must not be nil if given." if init.nil? raise "has_internal_id needs to be defined on association." unless self.reflect_on_association(scope) - before_validation :"track_#{scope}_#{column}!", on: :create, if: track_if - before_validation :"ensure_#{scope}_#{column}!", on: :create, if: ensure_if + init = infer_init(scope) if init == :not_given + before_validation :"track_#{scope}_#{column}!", on: hook_names, if: track_if + before_validation :"ensure_#{scope}_#{column}!", on: hook_names, if: ensure_if validates column, presence: presence + define_singleton_internal_id_methods(scope, column, init) + define_instance_internal_id_methods(scope, column, init, backfill) + end + + private + + def infer_init(scope) + case scope + when :project + AtomicInternalId.project_init(self) + when :group + AtomicInternalId.group_init(self) + else + # We require init here to retain the ability to recalculate in the absence of a + # InternalId record (we may delete records in `internal_ids` for example). + raise "has_internal_id - cannot infer init for scope: #{scope}" + end + end + + # Defines instance methods: + # - ensure_{scope}_{column}! + # - track_{scope}_{column}! + # - reset_{scope}_{column} + # - {column}= + def define_instance_internal_id_methods(scope, column, init, backfill) define_method("ensure_#{scope}_#{column}!") do return if backfill && self.class.where(column => nil).exists? @@ -103,19 +129,95 @@ module AtomicInternalId read_attribute(column) end end + + # Defines class methods: + # + # - with_{scope}_{column}_supply + # This method can be used to allocate a block of IID values during + # bulk operations (importing/copying, etc). This can be more efficient + # than creating instances one-by-one. + # + # Pass in a block that receives a `Supply` instance. To allocate a new + # IID value, call `Supply#next_value`. + # + # Example: + # + # MyClass.with_project_iid_supply(project) do |supply| + # attributes = MyClass.where(project: project).find_each do |record| + # record.attributes.merge(iid: supply.next_value) + # end + # + # bulk_insert(attributes) + # end + def define_singleton_internal_id_methods(scope, column, init) + define_singleton_method("with_#{scope}_#{column}_supply") do |scope_value, &block| + subject = find_by(scope => scope_value) || self + scope_attrs = ::AtomicInternalId.scope_attrs(scope_value) + usage = ::AtomicInternalId.scope_usage(self) + + generator = InternalId::InternalIdGenerator.new(subject, scope_attrs, usage, init) + + generator.with_lock do + supply = Supply.new(generator.record.last_value) + block.call(supply) + ensure + generator.track_greatest(supply.current_value) if supply + end + end + end + end + + def self.scope_attrs(scope_value) + { scope_value.class.table_name.singularize.to_sym => scope_value } if scope_value end def internal_id_scope_attrs(scope) scope_value = internal_id_read_scope(scope) - { scope_value.class.table_name.singularize.to_sym => scope_value } if scope_value + ::AtomicInternalId.scope_attrs(scope_value) end def internal_id_scope_usage - self.class.table_name.to_sym + ::AtomicInternalId.scope_usage(self.class) + end + + def self.scope_usage(including_class) + including_class.table_name.to_sym + end + + def self.project_init(klass, column_name = :iid) + ->(instance, scope) do + if instance + klass.where(project_id: instance.project_id).maximum(column_name) + elsif scope.present? + klass.where(**scope).maximum(column_name) + end + end + end + + def self.group_init(klass, column_name = :iid) + ->(instance, scope) do + if instance + klass.where(group_id: instance.group_id).maximum(column_name) + elsif scope.present? + klass.where(group: scope[:namespace]).maximum(column_name) + end + end end def internal_id_read_scope(scope) association(scope).reader end + + class Supply + attr_reader :current_value + + def initialize(start_value) + @current_value = start_value + end + + def next_value + @current_value += 1 + end + end end diff --git a/app/models/concerns/enums/ci/pipeline.rb b/app/models/concerns/enums/ci/pipeline.rb index f1bc43a12d8..bb8df37f649 100644 --- a/app/models/concerns/enums/ci/pipeline.rb +++ b/app/models/concerns/enums/ci/pipeline.rb @@ -53,6 +53,10 @@ module Enums sources.except(*dangling_sources.keys) end + def self.ci_and_parent_sources + ci_sources.merge(sources.slice(:parent_pipeline)) + end + # Returns the `Hash` to use for creating the `config_sources` enum for # `Ci::Pipeline`. def self.config_sources diff --git a/app/models/concerns/enums/internal_id.rb b/app/models/concerns/enums/internal_id.rb index 2d51d232e93..f01bd60ef16 100644 --- a/app/models/concerns/enums/internal_id.rb +++ b/app/models/concerns/enums/internal_id.rb @@ -14,7 +14,8 @@ module Enums operations_feature_flags: 6, operations_user_lists: 7, alert_management_alerts: 8, - sprints: 9 # iterations + sprints: 9, # iterations + design_management_designs: 10 } end end diff --git a/app/models/concerns/featurable.rb b/app/models/concerns/featurable.rb index 60aa46ce04c..20b72957ec2 100644 --- a/app/models/concerns/featurable.rb +++ b/app/models/concerns/featurable.rb @@ -37,7 +37,8 @@ module Featurable class_methods do def set_available_features(available_features = []) - @available_features = available_features + @available_features ||= [] + @available_features += available_features class_eval do available_features.each do |feature| diff --git a/app/models/concerns/from_union.rb b/app/models/concerns/from_union.rb index e25d603b802..be6744f1b2a 100644 --- a/app/models/concerns/from_union.rb +++ b/app/models/concerns/from_union.rb @@ -37,27 +37,6 @@ module FromUnion # rubocop: disable Gitlab/Union extend FromSetOperator define_set_operator Gitlab::SQL::Union - - alias_method :from_union_set_operator, :from_union - def from_union(members, remove_duplicates: true, alias_as: table_name) - if Feature.enabled?(:sql_set_operators) - from_union_set_operator(members, remove_duplicates: remove_duplicates, alias_as: alias_as) - else - # The original from_union method. - standard_from_union(members, remove_duplicates: remove_duplicates, alias_as: alias_as) - end - end - - private - - def standard_from_union(members, remove_duplicates: true, alias_as: table_name) - union = Gitlab::SQL::Union - .new(members, remove_duplicates: remove_duplicates) - .to_sql - - from(Arel.sql("(#{union}) #{alias_as}")) - end - # rubocop: enable Gitlab/Union end end diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb index 978a54bdee7..3dea4a9f5fb 100644 --- a/app/models/concerns/has_repository.rb +++ b/app/models/concerns/has_repository.rb @@ -109,6 +109,11 @@ module HasRepository Gitlab::RepositoryUrlBuilder.build(repository.full_path, protocol: :http) end + # Is overridden in EE::Project for Geo support + def lfs_http_url_to_repo(_operation = nil) + http_url_to_repo + end + def web_url(only_path: nil) Gitlab::UrlBuilder.build(self, only_path: only_path) end diff --git a/app/models/concerns/issue_available_features.rb b/app/models/concerns/issue_available_features.rb index 6efb8103b7b..886db133a94 100644 --- a/app/models/concerns/issue_available_features.rb +++ b/app/models/concerns/issue_available_features.rb @@ -6,18 +6,25 @@ module IssueAvailableFeatures extend ActiveSupport::Concern - # EE only features are listed on EE::IssueAvailableFeatures - def available_features_for_issue_types - {}.with_indifferent_access + class_methods do + # EE only features are listed on EE::IssueAvailableFeatures + def available_features_for_issue_types + {}.with_indifferent_access + end + end + + included do + scope :with_feature, ->(feature) { where(issue_type: available_features_for_issue_types[feature]) } end def issue_type_supports?(feature) - unless available_features_for_issue_types.has_key?(feature) + unless self.class.available_features_for_issue_types.has_key?(feature) raise ArgumentError, 'invalid feature' end - available_features_for_issue_types[feature].include?(issue_type) + self.class.available_features_for_issue_types[feature].include?(issue_type) end end IssueAvailableFeatures.prepend_if_ee('EE::IssueAvailableFeatures') +IssueAvailableFeatures::ClassMethods.prepend_if_ee('EE::IssueAvailableFeatures::ClassMethods') diff --git a/app/models/concerns/mentionable/reference_regexes.rb b/app/models/concerns/mentionable/reference_regexes.rb index 307d58a3a3c..5a5ce1809d0 100644 --- a/app/models/concerns/mentionable/reference_regexes.rb +++ b/app/models/concerns/mentionable/reference_regexes.rb @@ -22,7 +22,7 @@ module Mentionable def self.default_pattern strong_memoize(:default_pattern) do issue_pattern = Issue.reference_pattern - link_patterns = Regexp.union([Issue, Commit, MergeRequest, Epic].map(&:link_reference_pattern).compact) + link_patterns = Regexp.union([Issue, Commit, MergeRequest, Epic, Vulnerability].map(&:link_reference_pattern).compact) reference_pattern(link_patterns, issue_pattern) end end diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index cedcf164a49..b69fb2931c3 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -88,3 +88,5 @@ module ProjectFeaturesCompatibility project_feature.__send__(:write_attribute, field, value) # rubocop:disable GitlabSecurity/PublicSend end end + +ProjectFeaturesCompatibility.prepend_if_ee('EE::ProjectFeaturesCompatibility') diff --git a/app/models/concerns/project_services_loggable.rb b/app/models/concerns/project_services_loggable.rb index fecd77cdc98..e5385435138 100644 --- a/app/models/concerns/project_services_loggable.rb +++ b/app/models/concerns/project_services_loggable.rb @@ -16,8 +16,8 @@ module ProjectServicesLoggable def build_message(message, params = {}) { service_class: self.class.name, - project_id: project.id, - project_path: project.full_path, + project_id: project&.id, + project_path: project&.full_path, message: message }.merge(params) end diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index d1e3d9b2aff..28dc3366e51 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -36,10 +36,12 @@ module ProtectedRefAccess HUMAN_ACCESS_LEVELS[self.access_level] end - # CE access levels are always role-based, - # where as EE allows groups and users too + def type + :role + end + def role? - true + type == :role end def check_access(user) diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index 71b976c6f11..a82cf338039 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -90,7 +90,7 @@ module Storage end def old_repository_storages - @old_repository_storage_paths ||= repository_storages + @old_repository_storage_paths ||= repository_storages(legacy_only: true) end def repository_storages(legacy_only: false) diff --git a/app/models/concerns/todoable.rb b/app/models/concerns/todoable.rb new file mode 100644 index 00000000000..d93ab463251 --- /dev/null +++ b/app/models/concerns/todoable.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# == Todoable concern +# +# Specify object types that supports todos. +# +# Used by Issue, MergeRequest, Design and Epic. +# +module Todoable +end diff --git a/app/models/concerns/triggerable_hooks.rb b/app/models/concerns/triggerable_hooks.rb index b64a9e4f70b..325a5531926 100644 --- a/app/models/concerns/triggerable_hooks.rb +++ b/app/models/concerns/triggerable_hooks.rb @@ -13,7 +13,9 @@ module TriggerableHooks job_hooks: :job_events, pipeline_hooks: :pipeline_events, wiki_page_hooks: :wiki_page_events, - deployment_hooks: :deployment_events + deployment_hooks: :deployment_events, + feature_flag_hooks: :feature_flag_events, + release_hooks: :releases_events }.freeze extend ActiveSupport::Concern diff --git a/app/models/container_expiration_policy.rb b/app/models/container_expiration_policy.rb index 641d244b665..0441a5f0f5b 100644 --- a/app/models/container_expiration_policy.rb +++ b/app/models/container_expiration_policy.rb @@ -5,6 +5,13 @@ class ContainerExpirationPolicy < ApplicationRecord include UsageStatistics include EachBatch + POLICY_PARAMS = %w[ + older_than + keep_n + name_regex + name_regex_keep + ].freeze + belongs_to :project, inverse_of: :container_expiration_policy delegate :container_repositories, to: :project @@ -14,14 +21,15 @@ class ContainerExpirationPolicy < ApplicationRecord validates :cadence, presence: true, inclusion: { in: ->(_) { self.cadence_options.stringify_keys } } validates :older_than, inclusion: { in: ->(_) { self.older_than_options.stringify_keys } }, allow_nil: true validates :keep_n, inclusion: { in: ->(_) { self.keep_n_options.keys } }, allow_nil: true + validates :name_regex, presence: true, if: :enabled? validates :name_regex, untrusted_regexp: true, if: :enabled? validates :name_regex_keep, untrusted_regexp: true, if: :enabled? scope :active, -> { where(enabled: true) } scope :preloaded, -> { preload(project: [:route]) } - def self.executable - runnable_schedules.where( + def self.with_container_repositories + where( 'EXISTS (?)', ContainerRepository.select(1) .where( @@ -67,4 +75,8 @@ class ContainerExpirationPolicy < ApplicationRecord def disable! update_attribute(:enabled, false) end + + def policy_params + attributes.slice(*POLICY_PARAMS) + end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index d97b8776085..4adbd37608f 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -3,6 +3,9 @@ class ContainerRepository < ApplicationRecord include Gitlab::Utils::StrongMemoize include Gitlab::SQL::Pattern + include EachBatch + + WAITING_CLEANUP_STATUSES = %i[cleanup_scheduled cleanup_unfinished].freeze belongs_to :project @@ -10,6 +13,7 @@ class ContainerRepository < ApplicationRecord validates :name, uniqueness: { scope: :project_id } enum status: { delete_scheduled: 0, delete_failed: 1 } + enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 } delegate :client, to: :registry @@ -24,7 +28,9 @@ class ContainerRepository < ApplicationRecord ContainerRepository .joins("INNER JOIN (#{project_scope.to_sql}) projects on projects.id=container_repositories.project_id") end + scope :for_project_id, ->(project_id) { where(project_id: project_id) } scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } + scope :waiting_for_cleanup, -> { where(expiration_policy_cleanup_status: WAITING_CLEANUP_STATUSES) } def self.exists_by_path?(path) where( diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index 643b4060ad6..ed22d4ba231 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -3,14 +3,21 @@ class CustomEmoji < ApplicationRecord belongs_to :namespace, inverse_of: :custom_emoji + belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id' + + # For now only external emoji are supported. See https://gitlab.com/gitlab-org/gitlab/-/issues/230467 + validates :external, inclusion: { in: [true] } + + validates :file, public_url: true, if: :external + validate :valid_emoji_name - validates :namespace, presence: true + validates :group, presence: true validates :name, uniqueness: { scope: [:namespace_id, :name] }, presence: true, length: { maximum: 36 }, - format: { with: /\A\w+\z/ } + format: { with: /\A([a-z0-9]+[-_]?)+[a-z0-9]+\z/ } private diff --git a/app/models/dependency_proxy.rb b/app/models/dependency_proxy.rb new file mode 100644 index 00000000000..510a304ff17 --- /dev/null +++ b/app/models/dependency_proxy.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module DependencyProxy + def self.table_name_prefix + 'dependency_proxy_' + end +end diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb new file mode 100644 index 00000000000..3a81112340a --- /dev/null +++ b/app/models/dependency_proxy/blob.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class DependencyProxy::Blob < ApplicationRecord + include FileStoreMounter + + belongs_to :group + + validates :group, presence: true + validates :file, presence: true + validates :file_name, presence: true + + mount_file_store_uploader DependencyProxy::FileUploader + + def self.total_size + sum(:size) + end + + def self.find_or_build(file_name) + find_or_initialize_by(file_name: file_name) + end +end diff --git a/app/models/dependency_proxy/group_setting.rb b/app/models/dependency_proxy/group_setting.rb new file mode 100644 index 00000000000..bcf09b27129 --- /dev/null +++ b/app/models/dependency_proxy/group_setting.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class DependencyProxy::GroupSetting < ApplicationRecord + belongs_to :group + + validates :group, presence: true + + default_value_for :enabled, true +end diff --git a/app/models/dependency_proxy/registry.rb b/app/models/dependency_proxy/registry.rb new file mode 100644 index 00000000000..471d5be2600 --- /dev/null +++ b/app/models/dependency_proxy/registry.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class DependencyProxy::Registry + AUTH_URL = 'https://auth.docker.io'.freeze + LIBRARY_URL = 'https://registry-1.docker.io/v2'.freeze + + class << self + def auth_url(image) + "#{AUTH_URL}/token?service=registry.docker.io&scope=repository:#{image_path(image)}:pull" + end + + def manifest_url(image, tag) + "#{LIBRARY_URL}/#{image_path(image)}/manifests/#{tag}" + end + + def blob_url(image, blob_sha) + "#{LIBRARY_URL}/#{image_path(image)}/blobs/#{blob_sha}" + end + + private + + def image_path(image) + if image.include?('/') + image + else + "library/#{image}" + end + end + end +end diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 793ea3c29c3..db5fd167781 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -6,9 +6,11 @@ class DeployKey < Key has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :deploy_keys_projects + has_many :protected_branch_push_access_levels, class_name: '::ProtectedBranch::PushAccessLevel' - scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where('deploy_keys_projects.project_id in (?)', projects) } - scope :are_public, -> { where(public: true) } + scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where(deploy_keys_projects: { project_id: projects }) } + scope :with_write_access, -> { joins(:deploy_keys_projects).merge(DeployKeysProject.with_write_access) } + scope :are_public, -> { where(public: true) } scope :with_projects, -> { includes(deploy_keys_projects: { project: [:route, namespace: :route] }) } ignore_column :can_push, remove_after: '2019-12-15', remove_with: '12.6' @@ -54,4 +56,11 @@ class DeployKey < Key def projects_with_write_access Project.with_route.where(id: deploy_keys_projects.with_write_access.select(:project_id)) end + + def self.with_write_access_for_project(project, deploy_key: nil) + query = in_projects(project).with_write_access + query = query.where(id: deploy_key) if deploy_key + + query + end end diff --git a/app/models/deploy_keys_project.rb b/app/models/deploy_keys_project.rb index a9cc56a7246..40c66d5bc4c 100644 --- a/app/models/deploy_keys_project.rb +++ b/app/models/deploy_keys_project.rb @@ -6,7 +6,6 @@ class DeployKeysProject < ApplicationRecord scope :without_project_deleted, -> { joins(:project).where(projects: { pending_delete: false }) } scope :in_project, ->(project) { where(project: project) } scope :with_write_access, -> { where(can_push: true) } - scope :with_deploy_keys, -> { includes(:deploy_key) } accepts_nested_attributes_for :deploy_key diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 9355d73fae9..5fa9f2ef9f9 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -54,6 +54,10 @@ class DeployToken < ApplicationRecord !revoked && !expired? end + def deactivated? + !active? + end + def scopes AVAILABLE_SCOPES.select { |token_scope| read_attribute(token_scope) } end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 2d0d98136ec..36ac1bdb236 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -21,9 +21,7 @@ class Deployment < ApplicationRecord has_one :deployment_cluster - has_internal_id :iid, scope: :project, track_if: -> { !importing? }, init: ->(s) do - Deployment.where(project: s.project).maximum(:iid) if s&.project - end + has_internal_id :iid, scope: :project, track_if: -> { !importing? } validates :sha, presence: true validates :ref, presence: true @@ -79,8 +77,6 @@ class Deployment < ApplicationRecord after_transition any => :running do |deployment| deployment.run_after_commit do - next unless Feature.enabled?(:ci_send_deployment_hook_when_start, deployment.project) - Deployments::ExecuteHooksWorker.perform_async(id) end end diff --git a/app/models/deployment_merge_request.rb b/app/models/deployment_merge_request.rb index b67f96906f5..64a578e16bf 100644 --- a/app/models/deployment_merge_request.rb +++ b/app/models/deployment_merge_request.rb @@ -14,7 +14,12 @@ class DeploymentMergeRequest < ApplicationRecord end def self.deployed_to(name) + # We filter by project ID again so the query uses the index on + # (project_id, name), instead of using the index on + # (name varchar_pattern_ops). This results in better performance on + # GitLab.com. where('environments.name = ?', name) + .where('environments.project_id = merge_requests.target_project_id') end def self.deployed_after(time) diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb index 62e4bd6cebc..f5e52c04944 100644 --- a/app/models/design_management/design.rb +++ b/app/models/design_management/design.rb @@ -2,6 +2,7 @@ module DesignManagement class Design < ApplicationRecord + include AtomicInternalId include Importable include Noteable include Gitlab::FileTypeDetection @@ -10,12 +11,15 @@ module DesignManagement include Mentionable include WhereComposite include RelativePositioning + include Todoable + include Participable belongs_to :project, inverse_of: :designs belongs_to :issue has_many :actions has_many :versions, through: :actions, class_name: 'DesignManagement::Version', inverse_of: :designs + has_many :authors, -> { distinct }, through: :versions, class_name: 'User' # This is a polymorphic association, so we can't count on FK's to delete the # data has_many :notes, as: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -23,6 +27,10 @@ module DesignManagement has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_internal_id :iid, scope: :project, presence: true, + hook_names: %i[create update], # Deal with old records + track_if: -> { !importing? } + validates :project, :filename, presence: true validates :issue, presence: true, unless: :importing? validates :filename, uniqueness: { scope: :issue_id }, length: { maximum: 255 } @@ -30,6 +38,9 @@ module DesignManagement alias_attribute :title, :filename + participant :authors + participant :notes_with_associations + # Pre-fetching scope to include the data necessary to construct a # reference using `to_reference`. scope :for_reference, -> { includes(issue: [{ project: [:route, :namespace] }]) } diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb index 55c9084caf2..49aec8b9720 100644 --- a/app/models/design_management/version.rb +++ b/app/models/design_management/version.rb @@ -43,10 +43,7 @@ module DesignManagement validates :sha, presence: true validates :sha, uniqueness: { case_sensitive: false, scope: :issue_id } validates :author, presence: true - # We are not validating the issue object as it incurs an extra query to fetch - # the record from the DB. Instead, we rely on the foreign key constraint to - # ensure referential integrity. - validates :issue_id, presence: true, unless: :importing? + validates :issue, presence: true, unless: :importing? sha_attribute :sha diff --git a/app/models/diff_viewer/image.rb b/app/models/diff_viewer/image.rb index 62a3446a7b6..fca6c664196 100644 --- a/app/models/diff_viewer/image.rb +++ b/app/models/diff_viewer/image.rb @@ -10,5 +10,13 @@ module DiffViewer self.binary = true self.switcher_icon = 'doc-image' self.switcher_title = _('image diff') + + def self.can_render?(diff_file, verify_binary: true) + # When both blobs are missing, we often still have a textual diff that can + # be displayed + return false if diff_file.old_blob.nil? && diff_file.new_blob.nil? + + super + end end end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 793cdb5dece..70aa02063cc 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -16,6 +16,7 @@ class Discussion :commit_id, :confidential?, :for_commit?, + :for_design?, :for_merge_request?, :noteable_ability_name, :to_ability_name, diff --git a/app/models/environment.rb b/app/models/environment.rb index 66613869915..deded3eeae0 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -305,6 +305,10 @@ class Environment < ApplicationRecord latest_opened_most_severe_alert.present? end + def has_running_deployments? + all_deployments.running.exists? + end + def metrics prometheus_adapter.query(:environment, self) if has_metrics_and_can_query? end @@ -395,7 +399,7 @@ class Environment < ApplicationRecord # Overrides ReactiveCaching default to activate limit checking behind a FF def reactive_cache_limit_enabled? - Feature.enabled?(:reactive_caching_limit_environment, project) + Feature.enabled?(:reactive_caching_limit_environment, project, default_enabled: true) end end diff --git a/app/models/experiment.rb b/app/models/experiment.rb index 25640385536..f179a1fc6ce 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -2,26 +2,17 @@ class Experiment < ApplicationRecord has_many :experiment_users - has_many :users, through: :experiment_users - has_many :control_group_users, -> { merge(ExperimentUser.control) }, through: :experiment_users, source: :user - has_many :experimental_group_users, -> { merge(ExperimentUser.experimental) }, through: :experiment_users, source: :user validates :name, presence: true, uniqueness: true, length: { maximum: 255 } def self.add_user(name, group_type, user) - experiment = find_or_create_by(name: name) + return unless experiment = find_or_create_by(name: name) - return unless experiment - return if experiment.experiment_users.where(user: user).exists? - - group_type == ::Gitlab::Experimentation::GROUP_CONTROL ? experiment.add_control_user(user) : experiment.add_experimental_user(user) - end - - def add_control_user(user) - control_group_users << user + experiment.record_user_and_group(user, group_type) end - def add_experimental_user(user) - experimental_group_users << user + # Create or update the recorded experiment_user row for the user in this experiment. + def record_user_and_group(user, group_type) + experiment_users.find_or_initialize_by(user: user).update!(group_type: group_type) end end diff --git a/app/models/experiment_user.rb b/app/models/experiment_user.rb index 1571b0c3439..e447becc1bd 100644 --- a/app/models/experiment_user.rb +++ b/app/models/experiment_user.rb @@ -1,10 +1,14 @@ # frozen_string_literal: true class ExperimentUser < ApplicationRecord + include ::Gitlab::Experimentation::GroupTypes + belongs_to :experiment belongs_to :user - enum group_type: { control: 0, experimental: 1 } + enum group_type: { GROUP_CONTROL => 0, GROUP_EXPERIMENTAL => 1 } + validates :experiment_id, presence: true + validates :user_id, presence: true validates :group_type, presence: true end diff --git a/app/models/group.rb b/app/models/group.rb index 74f7efd253d..3509299a579 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -71,6 +71,9 @@ class Group < Namespace has_many :group_deploy_tokens has_many :deploy_tokens, through: :group_deploy_tokens + has_one :dependency_proxy_setting, class_name: 'DependencyProxy::GroupSetting' + has_many :dependency_proxy_blobs, class_name: 'DependencyProxy::Blob' + accepts_nested_attributes_for :variables, allow_destroy: true validate :visibility_level_allowed_by_projects @@ -98,6 +101,19 @@ class Group < Namespace scope :by_id, ->(groups) { where(id: groups) } + scope :for_authorized_group_members, -> (user_ids) do + joins(:group_members) + .where("members.user_id IN (?)", user_ids) + .where("access_level >= ?", Gitlab::Access::GUEST) + end + + scope :for_authorized_project_members, -> (user_ids) do + joins(projects: :project_authorizations) + .where("project_authorizations.user_id IN (?)", user_ids) + end + + delegate :default_branch_name, to: :namespace_settings + class << self def sort_by_attribute(method) if method == 'storage_size_desc' @@ -190,6 +206,10 @@ class Group < Namespace ::Gitlab.config.packages.enabled end + def dependency_proxy_feature_available? + ::Gitlab.config.dependency_proxy.enabled + end + def notification_email_for(user) # Finds the closest notification_setting with a `notification_email` notification_settings = notification_settings_for(user, hierarchy_order: :asc) @@ -571,12 +591,16 @@ class Group < Namespace ancestor_settings.allow_mfa_for_subgroups end + def has_project_with_service_desk_enabled? + Gitlab::ServiceDesk.supported? && all_projects.service_desk_enabled.exists? + end + private def update_two_factor_requirement return unless saved_change_to_require_two_factor_authentication? || saved_change_to_two_factor_grace_period? - members_with_descendants.find_each(&:update_two_factor_requirement) + direct_and_indirect_members.find_each(&:update_two_factor_requirement) end def path_changed_hook diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index 2d1bdecc770..b625a70b444 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -18,7 +18,9 @@ class ProjectHook < WebHook :job_hooks, :pipeline_hooks, :wiki_page_hooks, - :deployment_hooks + :deployment_hooks, + :feature_flag_hooks, + :release_hooks ] belongs_to :project diff --git a/app/models/instance_metadata.rb b/app/models/instance_metadata.rb new file mode 100644 index 00000000000..96622d0b1b3 --- /dev/null +++ b/app/models/instance_metadata.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class InstanceMetadata + attr_reader :version, :revision + + def initialize(version: Gitlab::VERSION, revision: Gitlab.revision) + @version = version + @revision = revision + end +end diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index 4c0469d849a..c735e593da7 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -61,13 +61,13 @@ class InternalId < ApplicationRecord class << self def track_greatest(subject, scope, usage, new_value, init) - InternalIdGenerator.new(subject, scope, usage) - .track_greatest(init, new_value) + InternalIdGenerator.new(subject, scope, usage, init) + .track_greatest(new_value) end def generate_next(subject, scope, usage, init) - InternalIdGenerator.new(subject, scope, usage) - .generate(init) + InternalIdGenerator.new(subject, scope, usage, init) + .generate end def reset(subject, scope, usage, value) @@ -99,15 +99,18 @@ class InternalId < ApplicationRecord # 4) In the absence of a record in the internal_ids table, one will be created # and last_value will be calculated on the fly. # - # subject: The instance we're generating an internal id for. Gets passed to init if called. + # subject: The instance or class we're generating an internal id for. # scope: Attributes that define the scope for id generation. + # Valid keys are `project/project_id` and `namespace/namespace_id`. # usage: Symbol to define the usage of the internal id, see InternalId.usages - attr_reader :subject, :scope, :scope_attrs, :usage + # init: Proc that accepts the subject and the scope and returns Integer|NilClass + attr_reader :subject, :scope, :scope_attrs, :usage, :init - def initialize(subject, scope, usage) + def initialize(subject, scope, usage, init = nil) @subject = subject @scope = scope @usage = usage + @init = init raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? @@ -119,13 +122,13 @@ class InternalId < ApplicationRecord # Generates next internal id and returns it # init: Block that gets called to initialize InternalId record if not present # Make sure to not throw exceptions in the absence of records (if this is expected). - def generate(init) + def generate subject.transaction do # Create a record in internal_ids if one does not yet exist # and increment its last value # # Note this will acquire a ROW SHARE lock on the InternalId record - (lookup || create_record(init)).increment_and_save! + record.increment_and_save! end end @@ -148,12 +151,20 @@ class InternalId < ApplicationRecord # and set its new_value if it is higher than the current last_value # # Note this will acquire a ROW SHARE lock on the InternalId record - def track_greatest(init, new_value) + def track_greatest(new_value) subject.transaction do - (lookup || create_record(init)).track_greatest_and_save!(new_value) + record.track_greatest_and_save!(new_value) end end + def record + @record ||= (lookup || create_record) + end + + def with_lock(&block) + record.with_lock(&block) + end + private # Retrieve InternalId record for (project, usage) combination, if it exists @@ -171,12 +182,16 @@ class InternalId < ApplicationRecord # was faster in doing this, we'll realize once we hit the unique key constraint # violation. We can safely roll-back the nested transaction and perform # a lookup instead to retrieve the record. - def create_record(init) + def create_record + raise ArgumentError, 'Cannot initialize without init!' unless init + + instance = subject.is_a?(::Class) ? nil : subject + subject.transaction(requires_new: true) do InternalId.create!( **scope, usage: usage_value, - last_value: init.call(subject) || 0 + last_value: init.call(instance, scope) || 0 ) end rescue ActiveRecord::RecordNotUnique diff --git a/app/models/issue.rb b/app/models/issue.rb index 5291b7890b6..7dc18cacd7c 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -21,6 +21,7 @@ class Issue < ApplicationRecord include IdInOrdered include Presentable include IssueAvailableFeatures + include Todoable DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze @@ -47,7 +48,7 @@ class Issue < ApplicationRecord belongs_to :moved_to, class_name: 'Issue' has_one :moved_from, class_name: 'Issue', foreign_key: :moved_to_id - has_internal_id :iid, scope: :project, track_if: -> { !importing? }, init: ->(s) { s&.project&.issues&.maximum(:iid) } + has_internal_id :iid, scope: :project, track_if: -> { !importing? } has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/issue_link.rb b/app/models/issue_link.rb index 9740b009396..5448ebdf50b 100644 --- a/app/models/issue_link.rb +++ b/app/models/issue_link.rb @@ -10,6 +10,7 @@ class IssueLink < ApplicationRecord validates :target, presence: true validates :source, uniqueness: { scope: :target_id, message: 'is already related' } validate :check_self_relation + validate :check_opposite_relation scope :for_source_issue, ->(issue) { where(source_id: issue.id) } scope :for_target_issue, ->(issue) { where(target_id: issue.id) } @@ -33,6 +34,14 @@ class IssueLink < ApplicationRecord errors.add(:source, 'cannot be related to itself') end end + + def check_opposite_relation + return unless source && target + + if IssueLink.find_by(source: target, target: source) + errors.add(:source, 'is already related to this issue') + end + end end IssueLink.prepend_if_ee('EE::IssueLink') diff --git a/app/models/issues/csv_import.rb b/app/models/issues/csv_import.rb new file mode 100644 index 00000000000..d141f126ec9 --- /dev/null +++ b/app/models/issues/csv_import.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class Issues::CsvImport < ApplicationRecord + self.table_name = 'csv_issue_imports' + + belongs_to :project, optional: false + belongs_to :user, optional: false +end diff --git a/app/models/iteration.rb b/app/models/iteration.rb index bd245de411c..ba7cd973e9d 100644 --- a/app/models/iteration.rb +++ b/app/models/iteration.rb @@ -17,8 +17,8 @@ class Iteration < ApplicationRecord belongs_to :project belongs_to :group - has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.iterations&.maximum(:iid) } - has_internal_id :iid, scope: :group, init: ->(s) { s&.group&.iterations&.maximum(:iid) } + has_internal_id :iid, scope: :project + has_internal_id :iid, scope: :group validates :start_date, presence: true validates :due_date, presence: true diff --git a/app/models/member.rb b/app/models/member.rb index 498e03b2c1a..687830f5267 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -96,6 +96,8 @@ class Member < ApplicationRecord scope :owners, -> { active.where(access_level: OWNER) } scope :owners_and_maintainers, -> { active.where(access_level: [OWNER, MAINTAINER]) } scope :with_user, -> (user) { where(user: user) } + scope :with_user_by_email, -> (email) { left_join_users.where(users: { email: email } ) } + scope :preload_user_and_notification_settings, -> { preload(user: :notification_settings) } scope :with_source_id, ->(source_id) { where(source_id: source_id) } @@ -417,6 +419,10 @@ class Member < ApplicationRecord invite? && user_id.nil? end + def created_by_name + created_by&.name + end + private def send_invite diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 34958936c9f..2bbcdbbe5ce 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -21,6 +21,7 @@ class GroupMember < Member scope :of_groups, ->(groups) { where(source_id: groups.select(:id)) } scope :of_ldap_type, -> { where(ldap: true) } scope :count_users_by_group_id, -> { group(:source_id).count } + scope :with_user, -> (user) { where(user: user) } after_create :update_two_factor_requirement, unless: :invite? after_destroy :update_two_factor_requirement, unless: :invite? diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 24541ba3218..d379f85bc15 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -22,6 +22,7 @@ class MergeRequest < ApplicationRecord include StateEventable include ApprovableBase include IdInOrdered + include Todoable extend ::Gitlab::Utils::Override @@ -40,7 +41,14 @@ class MergeRequest < ApplicationRecord belongs_to :merge_user, class_name: "User" belongs_to :iteration, foreign_key: 'sprint_id' - has_internal_id :iid, scope: :target_project, track_if: -> { !importing? }, init: ->(s) { s&.target_project&.merge_requests&.maximum(:iid) } + has_internal_id :iid, scope: :target_project, track_if: -> { !importing? }, + init: ->(mr, scope) do + if mr + mr.target_project&.merge_requests&.maximum(:iid) + elsif scope[:project] + where(target_project: scope[:project]).maximum(:iid) + end + end has_many :merge_request_diffs has_many :merge_request_context_commits, inverse_of: :merge_request @@ -48,6 +56,7 @@ class MergeRequest < ApplicationRecord has_one :merge_request_diff, -> { order('merge_request_diffs.id DESC') }, inverse_of: :merge_request + has_one :cleanup_schedule, inverse_of: :merge_request belongs_to :latest_merge_request_diff, class_name: 'MergeRequestDiff' manual_inverse_association :latest_merge_request_diff, :merge_request @@ -293,6 +302,7 @@ class MergeRequest < ApplicationRecord scope :preload_author, -> { preload(:author) } scope :preload_approved_by_users, -> { preload(:approved_by_users) } scope :preload_metrics, -> (relation) { preload(metrics: relation) } + scope :with_web_entity_associations, -> { preload(:author, :target_project) } scope :with_auto_merge_enabled, -> do with_state(:opened).where(auto_merge_enabled: true) @@ -302,6 +312,8 @@ class MergeRequest < ApplicationRecord includes(:metrics) end + scope :with_jira_issue_keys, -> { where('title ~ :regex OR merge_requests.description ~ :regex', regex: Gitlab::Regex.jira_issue_key_regex.source) } + after_save :keep_around_commit, unless: :importing? alias_attribute :project, :target_project diff --git a/app/models/merge_request/cleanup_schedule.rb b/app/models/merge_request/cleanup_schedule.rb new file mode 100644 index 00000000000..79817269be2 --- /dev/null +++ b/app/models/merge_request/cleanup_schedule.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class MergeRequest::CleanupSchedule < ApplicationRecord + belongs_to :merge_request, inverse_of: :cleanup_schedule + + validates :scheduled_at, presence: true + + def self.scheduled_merge_request_ids(limit) + where('completed_at IS NULL AND scheduled_at <= NOW()') + .order('scheduled_at DESC') + .limit(limit) + .pluck(:merge_request_id) + end +end diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index 55ff4250c2d..817e77bf12f 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -8,6 +8,10 @@ class MergeRequestDiffFile < ApplicationRecord belongs_to :merge_request_diff, inverse_of: :merge_request_diff_files alias_attribute :index, :relative_order + scope :by_paths, ->(paths) do + where("new_path in (?) OR old_path in (?)", paths, paths) + end + def utf8_diff return '' if diff.blank? diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 0a315ba8db2..c8776be5e4a 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -12,8 +12,8 @@ class Milestone < ApplicationRecord has_many :milestone_releases has_many :releases, through: :milestone_releases - has_internal_id :iid, scope: :project, track_if: -> { !importing? }, init: ->(s) { s&.project&.milestones&.maximum(:iid) } - has_internal_id :iid, scope: :group, track_if: -> { !importing? }, init: ->(s) { s&.group&.milestones&.maximum(:iid) } + has_internal_id :iid, scope: :project, track_if: -> { !importing? } + has_internal_id :iid, scope: :group, track_if: -> { !importing? } has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/namespace.rb b/app/models/namespace.rb index fd31042c2f6..232d0a6b05d 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -96,7 +96,8 @@ class Namespace < ApplicationRecord 'COALESCE(SUM(ps.snippets_size), 0) AS snippets_size', 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', - 'COALESCE(SUM(ps.packages_size), 0) AS packages_size' + 'COALESCE(SUM(ps.packages_size), 0) AS packages_size', + 'COALESCE(SUM(ps.uploads_size), 0) AS uploads_size' ) end @@ -117,8 +118,12 @@ class Namespace < ApplicationRecord # query - The search query as a String. # # Returns an ActiveRecord::Relation. - def search(query) - fuzzy_search(query, [:name, :path]) + def search(query, include_parents: false) + if include_parents + where(id: Route.for_routable_type(Namespace.name).fuzzy_search(query, [Route.arel_table[:path], Route.arel_table[:name]]).select(:source_id)) + else + fuzzy_search(query, [:path, :name]) + end end def clean_path(path) @@ -284,7 +289,8 @@ class Namespace < ApplicationRecord # that belongs to this namespace def all_projects if Feature.enabled?(:recursive_approach_for_all_projects) - Project.where(namespace: self_and_descendants) + namespace = user? ? self : self_and_descendants + Project.where(namespace: namespace) else Project.inside_path(full_path) end @@ -357,7 +363,7 @@ class Namespace < ApplicationRecord def pages_virtual_domain Pages::VirtualDomain.new( - all_projects_with_pages.includes(:route, :project_feature), + all_projects_with_pages.includes(:route, :project_feature, pages_metadatum: :pages_deployment), trim_prefix: full_path ) end @@ -388,7 +394,6 @@ class Namespace < ApplicationRecord end def changing_shared_runners_enabled_is_allowed - return unless Feature.enabled?(:disable_shared_runners_on_group, default_enabled: true) return unless new_record? || changes.has_key?(:shared_runners_enabled) if shared_runners_enabled && has_parent? && parent.shared_runners_setting == 'disabled_and_unoverridable' @@ -397,7 +402,6 @@ class Namespace < ApplicationRecord end def changing_allow_descendants_override_disabled_shared_runners_is_allowed - return unless Feature.enabled?(:disable_shared_runners_on_group, default_enabled: true) return unless new_record? || changes.has_key?(:allow_descendants_override_disabled_shared_runners) if shared_runners_enabled && !new_record? diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index 5723a823e98..a3df82998c4 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -11,6 +11,7 @@ class Namespace::RootStorageStatistics < ApplicationRecord packages_size #{SNIPPETS_SIZE_STAT_NAME} pipeline_artifacts_size + uploads_size ).freeze self.primary_key = :namespace_id @@ -50,7 +51,8 @@ class Namespace::RootStorageStatistics < ApplicationRecord 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', 'COALESCE(SUM(ps.packages_size), 0) AS packages_size', "COALESCE(SUM(ps.snippets_size), 0) AS #{SNIPPETS_SIZE_STAT_NAME}", - 'COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size' + 'COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size', + 'COALESCE(SUM(ps.uploads_size), 0) AS uploads_size' ) end diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index 6f31208f28b..50844403d7f 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -6,10 +6,18 @@ class NamespaceSetting < ApplicationRecord validate :default_branch_name_content validate :allow_mfa_for_group + before_validation :normalize_default_branch_name + NAMESPACE_SETTINGS_PARAMS = [:default_branch_name].freeze self.primary_key = :namespace_id + private + + def normalize_default_branch_name + self.default_branch_name = nil if default_branch_name.blank? + end + def default_branch_name_content return if default_branch_name.nil? diff --git a/app/models/note.rb b/app/models/note.rb index 954843505d4..cfdac6c432f 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -197,8 +197,8 @@ class Note < ApplicationRecord .map(&:position) end - def count_for_collection(ids, type) - user.select('noteable_id', 'COUNT(*) as count') + def count_for_collection(ids, type, count_column = 'COUNT(*) as count') + user.select(:noteable_id, count_column) .group(:noteable_id) .where(noteable_type: type, noteable_id: ids) end diff --git a/app/models/operations/feature_flag.rb b/app/models/operations/feature_flag.rb index 104338b80d1..442f9d36c43 100644 --- a/app/models/operations/feature_flag.rb +++ b/app/models/operations/feature_flag.rb @@ -2,6 +2,7 @@ module Operations class FeatureFlag < ApplicationRecord + include AfterCommitQueue include AtomicInternalId include IidRoutes include Limitable @@ -12,7 +13,7 @@ module Operations belongs_to :project - has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.operations_feature_flags&.maximum(:iid) } + has_internal_id :iid, scope: :project default_value_for :active, true @@ -77,6 +78,22 @@ module Operations Ability.issues_readable_by_user(issues, current_user) end + def execute_hooks(current_user) + run_after_commit do + feature_flag_data = Gitlab::DataBuilder::FeatureFlag.build(self, current_user) + project.execute_hooks(feature_flag_data, :feature_flag_hooks) + end + end + + def hook_attrs + { + id: id, + name: name, + description: description, + active: active + } + end + private def version_associations diff --git a/app/models/operations/feature_flags/user_list.rb b/app/models/operations/feature_flags/user_list.rb index b9bdcb59d5f..3e492eaa892 100644 --- a/app/models/operations/feature_flags/user_list.rb +++ b/app/models/operations/feature_flags/user_list.rb @@ -5,6 +5,7 @@ module Operations class UserList < ApplicationRecord include AtomicInternalId include IidRoutes + include ::Gitlab::SQL::Pattern self.table_name = 'operations_user_lists' @@ -12,7 +13,7 @@ module Operations has_many :strategy_user_lists has_many :strategies, through: :strategy_user_lists - has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.operations_feature_flags_user_lists&.maximum(:iid) }, presence: true + has_internal_id :iid, scope: :project, presence: true validates :project, presence: true validates :name, @@ -23,6 +24,10 @@ module Operations before_destroy :ensure_no_associated_strategies + scope :for_name_like, -> (query) do + fuzzy_search(query, [:name], use_minimum_char_limit: false) + end + private def ensure_no_associated_strategies diff --git a/app/models/packages/build_info.rb b/app/models/packages/build_info.rb index df8cf68490e..1b0f0ed8ffd 100644 --- a/app/models/packages/build_info.rb +++ b/app/models/packages/build_info.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true class Packages::BuildInfo < ApplicationRecord - belongs_to :package, inverse_of: :build_info + belongs_to :package, inverse_of: :build_infos belongs_to :pipeline, class_name: 'Ci::Pipeline' end diff --git a/app/models/packages/event.rb b/app/models/packages/event.rb index f1d0af64ccd..959c94931ec 100644 --- a/app/models/packages/event.rb +++ b/app/models/packages/event.rb @@ -3,6 +3,7 @@ class Packages::Event < ApplicationRecord belongs_to :package, optional: true + UNIQUE_EVENTS_ALLOWED = %i[push_package delete_package pull_package].freeze EVENT_SCOPES = ::Packages::Package.package_types.merge(container: 1000, tag: 1001).freeze enum event_scope: EVENT_SCOPES @@ -22,4 +23,20 @@ class Packages::Event < ApplicationRecord } enum originator_type: { user: 0, deploy_token: 1, guest: 2 } + + def self.allowed_event_name(event_scope, event_type, originator) + return unless event_allowed?(event_scope, event_type, originator) + + # remove `package` from the event name to avoid issues with HLLRedisCounter class parsing + "i_package_#{event_scope}_#{originator}_#{event_type.gsub(/_packages?/, "")}" + end + + # Remove some of the events, for now, so we don't hammer Redis too hard. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/280770 + def self.event_allowed?(event_scope, event_type, originator) + return false if originator.to_sym == :guest + return true if UNIQUE_EVENTS_ALLOWED.include?(event_type.to_sym) + + false + end end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index b8f8d45ff62..60aab0a7222 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -3,6 +3,7 @@ class Packages::Package < ApplicationRecord include Sortable include Gitlab::SQL::Pattern include UsageStatistics + include Gitlab::Utils::StrongMemoize belongs_to :project belongs_to :creator, class_name: 'User' @@ -16,7 +17,8 @@ class Packages::Package < ApplicationRecord has_one :maven_metadatum, inverse_of: :package, class_name: 'Packages::Maven::Metadatum' has_one :nuget_metadatum, inverse_of: :package, class_name: 'Packages::Nuget::Metadatum' has_one :composer_metadatum, inverse_of: :package, class_name: 'Packages::Composer::Metadatum' - has_one :build_info, inverse_of: :package + has_many :build_infos, inverse_of: :package + has_many :pipelines, through: :build_infos accepts_nested_attributes_for :conan_metadatum accepts_nested_attributes_for :maven_metadatum @@ -38,12 +40,13 @@ class Packages::Package < ApplicationRecord validates :name, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? validates :name, format: { with: Gitlab::Regex.generic_package_name_regex }, if: :generic? validates :name, format: { with: Gitlab::Regex.nuget_package_name_regex }, if: :nuget? - validates :version, format: { with: Gitlab::Regex.semver_regex }, if: :npm? validates :version, format: { with: Gitlab::Regex.nuget_version_regex }, if: :nuget? validates :version, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? validates :version, format: { with: Gitlab::Regex.maven_version_regex }, if: -> { version? && maven? } validates :version, format: { with: Gitlab::Regex.pypi_version_regex }, if: :pypi? validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :golang? + validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { composer_tag_version? || npm? } + validates :version, presence: true, format: { with: Gitlab::Regex.generic_package_version_regex }, @@ -58,7 +61,7 @@ class Packages::Package < ApplicationRecord scope :with_version, ->(version) { where(version: version) } scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } scope :with_package_type, ->(package_type) { where(package_type: package_type) } - scope :including_build_info, -> { includes(build_info: { pipeline: :user }) } + scope :including_build_info, -> { includes(pipelines: :user) } scope :including_project_route, -> { includes(project: { namespace: :route }) } scope :including_tags, -> { includes(:tags) } @@ -165,8 +168,16 @@ class Packages::Package < ApplicationRecord .order(:version) end + # Technical debt: to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/281937 + def original_build_info + strong_memoize(:original_build_info) do + build_infos.first + end + end + + # Technical debt: to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/281937 def pipeline - build_info&.pipeline + original_build_info&.pipeline end def tag_names @@ -175,6 +186,10 @@ class Packages::Package < ApplicationRecord private + def composer_tag_version? + composer? && !Gitlab::Regex.composer_dev_version_regex.match(version.to_s) + end + def valid_conan_package_recipe recipe_exists = project.packages .conan diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index 4ebd96797db..d68f75140ac 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -8,6 +8,8 @@ class Packages::PackageFile < ApplicationRecord belongs_to :package has_one :conan_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Conan::FileMetadatum' + has_many :package_file_build_infos, inverse_of: :package_file, class_name: 'Packages::PackageFileBuildInfo' + has_many :pipelines, through: :package_file_build_infos accepts_nested_attributes_for :conan_file_metadatum diff --git a/app/models/packages/package_file_build_info.rb b/app/models/packages/package_file_build_info.rb new file mode 100644 index 00000000000..5cabed446aa --- /dev/null +++ b/app/models/packages/package_file_build_info.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class Packages::PackageFileBuildInfo < ApplicationRecord + belongs_to :package_file, inverse_of: :package_file_build_infos + belongs_to :pipeline, class_name: 'Ci::Pipeline' +end diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index 84d820e539c..9855731778f 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -22,11 +22,7 @@ module Pages end def source - if artifacts_archive && !artifacts_archive.file_storage? - zip_source - else - file_source - end + zip_source || file_source end def prefix @@ -42,18 +38,36 @@ module Pages attr_reader :project, :trim_prefix, :domain def artifacts_archive - return unless Feature.enabled?(:pages_artifacts_archive, project) + return unless Feature.enabled?(:pages_serve_from_artifacts_archive, project) + + project.pages_metadatum.artifacts_archive + end + + def deployment + return unless Feature.enabled?(:pages_serve_from_deployments, project) - # Using build artifacts is temporary solution for quick test - # in production environment, we'll replace this with proper - # `pages_deployments` later - project.pages_metadatum.artifacts_archive&.file + project.pages_metadatum.pages_deployment end def zip_source + source = deployment || artifacts_archive + + return unless source&.file + + return if source.file.file_storage? && !Feature.enabled?(:pages_serve_with_zip_file_protocol, project) + + # artifacts archive doesn't support this + file_count = source.file_count if source.respond_to?(:file_count) + + global_id = ::Gitlab::GlobalId.build(source, id: source.id).to_s + { type: 'zip', - path: artifacts_archive.url(expire_at: 1.day.from_now) + path: source.file.url_or_file_path(expire_at: 1.day.from_now), + global_id: global_id, + sha256: source.file_sha256, + file_size: source.size, + file_count: file_count } end diff --git a/app/models/pages_deployment.rb b/app/models/pages_deployment.rb index cd952c32046..61818a63764 100644 --- a/app/models/pages_deployment.rb +++ b/app/models/pages_deployment.rb @@ -4,19 +4,27 @@ class PagesDeployment < ApplicationRecord include FileStoreMounter + attribute :file_store, :integer, default: -> { ::Pages::DeploymentUploader.default_store } + belongs_to :project, optional: false belongs_to :ci_build, class_name: 'Ci::Build', optional: true + scope :older_than, -> (id) { where('id < ?', id) } + validates :file, presence: true validates :file_store, presence: true, inclusion: { in: ObjectStorage::SUPPORTED_STORES } validates :size, presence: true, numericality: { greater_than: 0, only_integer: true } + validates :file_count, presence: true, numericality: { greater_than_or_equal_to: 0, only_integer: true } + validates :file_sha256, presence: true before_validation :set_size, if: :file_changed? - default_value_for(:file_store) { ::Pages::DeploymentUploader.default_store } - mount_file_store_uploader ::Pages::DeploymentUploader + def log_geo_deleted_event + # this is to be adressed in https://gitlab.com/groups/gitlab-org/-/epics/589 + end + private def set_size diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 98db47deaa3..8192310ddfb 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -286,7 +286,7 @@ class PagesDomain < ApplicationRecord return unless domain if domain.downcase.ends_with?(Settings.pages.host.downcase) - self.errors.add(:domain, "*.#{Settings.pages.host} is restricted") + self.errors.add(:domain, "*.#{Settings.pages.host} is restricted. Please compare our documentation at https://docs.gitlab.com/ee/administration/pages/#advanced-configuration against your configuration.") end end diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index e01cb0530a5..5aa5f2c842b 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -26,6 +26,7 @@ class PersonalAccessToken < ApplicationRecord scope :revoked, -> { where(revoked: true) } scope :not_revoked, -> { where(revoked: [false, nil]) } scope :for_user, -> (user) { where(user: user) } + scope :for_users, -> (users) { where(user: users) } scope :preload_users, -> { preload(:user) } scope :order_expires_at_asc, -> { reorder(expires_at: :asc) } scope :order_expires_at_desc, -> { reorder(expires_at: :desc) } diff --git a/app/models/project.rb b/app/models/project.rb index dbedd6d120c..ebd8e56246d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -346,7 +346,8 @@ class Project < ApplicationRecord # GitLab Pages has_many :pages_domains has_one :pages_metadatum, class_name: 'ProjectPagesMetadatum', inverse_of: :project - has_many :pages_deployments + # we need to clean up files, not only remove records + has_many :pages_deployments, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Can be too many records. We need to implement delete_all in batches. # Issue https://gitlab.com/gitlab-org/gitlab/-/issues/228637 @@ -378,7 +379,7 @@ class Project < ApplicationRecord delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, :forking_enabled?, :issues_enabled?, - :pages_enabled?, :public_pages?, :private_pages?, + :pages_enabled?, :snippets_enabled?, :public_pages?, :private_pages?, :merge_requests_access_level, :forking_access_level, :issues_access_level, :wiki_access_level, :snippets_access_level, :builds_access_level, :repository_access_level, :pages_access_level, :metrics_dashboard_access_level, @@ -570,6 +571,7 @@ class Project < ApplicationRecord scope :imported_from, -> (type) { where(import_type: type) } scope :with_tracing_enabled, -> { joins(:tracing_setting) } + scope :with_enabled_error_tracking, -> { joins(:error_tracking_setting).where(project_error_tracking_settings: { enabled: true }) } enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } @@ -600,7 +602,7 @@ class Project < ApplicationRecord # Returns a collection of projects that is either public or visible to the # logged in user. def self.public_or_visible_to_user(user = nil, min_access_level = nil) - min_access_level = nil if user&.admin? + min_access_level = nil if user&.can_read_all_resources? return public_to_user unless user @@ -626,7 +628,7 @@ class Project < ApplicationRecord def self.with_feature_available_for_user(feature, user) visible = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC] - if user&.admin? + if user&.can_read_all_resources? with_feature_enabled(feature) elsif user min_access_level = ProjectFeature.required_minimum_access_level(feature) @@ -1193,7 +1195,6 @@ class Project < ApplicationRecord end def changing_shared_runners_enabled_is_allowed - return unless Feature.enabled?(:disable_shared_runners_on_group, default_enabled: true) return unless new_record? || changes.has_key?(:shared_runners_enabled) if shared_runners_enabled && group && group.shared_runners_setting == 'disabled_and_unoverridable' @@ -1340,8 +1341,7 @@ class Project < ApplicationRecord end def find_or_initialize_services - available_services_names = - Service.available_services_names + Service.project_specific_services_names - disabled_services + available_services_names = Service.available_services_names - disabled_services available_services_names.map do |service_name| find_or_initialize_service(service_name) @@ -1468,11 +1468,6 @@ class Project < ApplicationRecord services.public_send(hooks_scope).any? # rubocop:disable GitlabSecurity/PublicSend end - # Is overridden in EE - def lfs_http_url_to_repo(_) - http_url_to_repo - end - def feature_usage super.presence || build_feature_usage end @@ -1801,6 +1796,8 @@ class Project < ApplicationRecord mark_pages_as_not_deployed unless destroyed? + DestroyPagesDeploymentsWorker.perform_async(id) + # 1. We rename pages to temporary directory # 2. We wait 5 minutes, due to NFS caching # 3. We asynchronously remove pages with force @@ -1817,7 +1814,11 @@ class Project < ApplicationRecord end def mark_pages_as_not_deployed - ensure_pages_metadatum.update!(deployed: false, artifacts_archive: nil) + ensure_pages_metadatum.update!(deployed: false, artifacts_archive: nil, pages_deployment: nil) + end + + def update_pages_deployment!(deployment) + ensure_pages_metadatum.update!(pages_deployment: deployment) end def write_repository_config(gl_full_path: full_path) @@ -2090,21 +2091,36 @@ class Project < ApplicationRecord (auto_devops || build_auto_devops)&.predefined_variables end - # Tries to set repository as read_only, checking for existing Git transfers in progress beforehand + RepositoryReadOnlyError = Class.new(StandardError) + + # Tries to set repository as read_only, checking for existing Git transfers in + # progress beforehand. Setting a repository read-only will fail if it is + # already in that state. # - # @return [Boolean] true when set to read_only or false when an existing git transfer is in progress + # @return nil. Failures will raise an exception def set_repository_read_only! with_lock do - break false if git_transfer_in_progress? + raise RepositoryReadOnlyError, _('Git transfer in progress') if + git_transfer_in_progress? - update_column(:repository_read_only, true) + raise RepositoryReadOnlyError, _('Repository already read-only') if + self.class.where(id: id).pick(:repository_read_only) + + raise ActiveRecord::RecordNotSaved, _('Database update failed') unless + update_column(:repository_read_only, true) + + nil end end - # Set repository as writable again + # Set repository as writable again. Unlike setting it read-only, this will + # succeed if the repository is already writable. def set_repository_writable! with_lock do - update_column(:repository_read_only, false) + raise ActiveRecord::RecordNotSaved, _('Database update failed') unless + update_column(:repository_read_only, false) + + nil end end diff --git a/app/models/project_repository_storage_move.rb b/app/models/project_repository_storage_move.rb index 76f428fe925..3429dbe3a85 100644 --- a/app/models/project_repository_storage_move.rb +++ b/app/models/project_repository_storage_move.rb @@ -46,8 +46,15 @@ class ProjectRepositoryStorageMove < ApplicationRecord transition replicated: :cleanup_failed end - after_transition initial: :scheduled do |storage_move| - storage_move.project.update_column(:repository_read_only, true) + around_transition initial: :scheduled do |storage_move, block| + block.call + + begin + storage_move.project.set_repository_read_only! + rescue => err + errors.add(:project, err.message) + next false + end storage_move.run_after_commit do ProjectUpdateRepositoryStorageWorker.perform_async( @@ -56,17 +63,18 @@ class ProjectRepositoryStorageMove < ApplicationRecord storage_move.id ) end + + true end - after_transition started: :replicated do |storage_move| - storage_move.project.update_columns( - repository_read_only: false, - repository_storage: storage_move.destination_storage_name - ) + before_transition started: :replicated do |storage_move| + storage_move.project.set_repository_writable! + + storage_move.project.update_column(:repository_storage, storage_move.destination_storage_name) end - after_transition started: :failed do |storage_move| - storage_move.project.update_column(:repository_read_only, false) + before_transition started: :failed do |storage_move| + storage_move.project.set_repository_writable! end state :initial, value: 1 diff --git a/app/models/project_services/alerts_service.rb b/app/models/project_services/alerts_service.rb index 28902114f3c..5b7d149ace1 100644 --- a/app/models/project_services/alerts_service.rb +++ b/app/models/project_services/alerts_service.rb @@ -14,6 +14,8 @@ class AlertsService < Service before_validation :prevent_token_assignment before_validation :ensure_token, if: :activated? + after_save :update_http_integration + def url return if instance? || template? @@ -77,6 +79,14 @@ class AlertsService < Service def url_helpers Gitlab::Routing.url_helpers end + + def update_http_integration + return unless project_id && type == 'AlertsService' + + AlertManagement::SyncAlertServiceDataService # rubocop: disable CodeReuse/ServiceClass + .new(self) + .execute + end end AlertsService.prepend_if_ee('EE::AlertsService') diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 732da62863f..7814bdb7106 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -30,7 +30,7 @@ class JiraService < IssueTrackerService # TODO: we can probably just delegate as part of # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :username, :password, :url, :api_url, :jira_issue_transition_id, :project_key, :issues_enabled + data_field :username, :password, :url, :api_url, :jira_issue_transition_id, :project_key, :issues_enabled, :vulnerabilities_enabled, :vulnerabilities_issuetype before_update :reset_password after_commit :update_deployment_type, on: [:create, :update], if: :update_deployment_type? diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 0d2f89fb18d..c11a7fea1c6 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -19,14 +19,14 @@ class ProjectStatistics < ApplicationRecord before_save :update_storage_size - COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size].freeze + COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size, :uploads_size].freeze INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size], pipeline_artifacts_size: %i[storage_size], snippets_size: %i[storage_size] }.freeze - NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size].freeze + NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size, :uploads_size].freeze scope :for_project_ids, ->(project_ids) { where(project_id: project_ids) } @@ -72,6 +72,12 @@ class ProjectStatistics < ApplicationRecord self.lfs_objects_size = project.lfs_objects.sum(:size) end + def update_uploads_size + return uploads_size unless Feature.enabled?(:count_uploads_size_in_storage_stats, project) + + self.uploads_size = project.uploads.sum(:size) + end + # `wiki_size` and `snippets_size` have no default value in the database # and the column can be nil. # This means that, when the columns were added, all rows had nil @@ -98,6 +104,10 @@ class ProjectStatistics < ApplicationRecord # might try to update project statistics before the `pipeline_artifacts_size` column has been created. storage_size += pipeline_artifacts_size if self.class.column_names.include?('pipeline_artifacts_size') + # The `uploads_size` column was added on 20201105021637 but db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb + # might try to update project statistics before the `uploads_size` column has been created. + storage_size += uploads_size if self.class.column_names.include?('uploads_size') + self.storage_size = storage_size end diff --git a/app/models/protected_branch/push_access_level.rb b/app/models/protected_branch/push_access_level.rb index bde1d29ad7f..63d577a4866 100644 --- a/app/models/protected_branch/push_access_level.rb +++ b/app/models/protected_branch/push_access_level.rb @@ -2,4 +2,29 @@ class ProtectedBranch::PushAccessLevel < ApplicationRecord include ProtectedBranchAccess + + belongs_to :deploy_key + + validates :access_level, uniqueness: { scope: :protected_branch_id, if: :role?, + conditions: -> { where(user_id: nil, group_id: nil, deploy_key_id: nil) } } + validates :deploy_key_id, uniqueness: { scope: :protected_branch_id, allow_nil: true } + validate :validate_deploy_key_membership + + def type + if self.deploy_key.present? + :deploy_key + else + super + end + end + + private + + def validate_deploy_key_membership + return unless deploy_key + + unless project.deploy_keys_projects.where(deploy_key: deploy_key).exists? + self.errors.add(:deploy_key, 'is not enabled for this project') + end + end end diff --git a/app/models/release.rb b/app/models/release.rb index f2162a0f674..c56df0a6aa3 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -83,6 +83,15 @@ class Release < ApplicationRecord self.milestones.map {|m| m.title }.sort.join(", ") end + def to_hook_data(action) + Gitlab::HookData::ReleaseBuilder.new(self).build(action) + end + + def execute_hooks(action) + hook_data = to_hook_data(action) + project.execute_hooks(hook_data, :release_hooks) + end + private def actual_sha diff --git a/app/models/releases/link.rb b/app/models/releases/link.rb index 82272f4857a..fc2fa639f56 100644 --- a/app/models/releases/link.rb +++ b/app/models/releases/link.rb @@ -30,5 +30,15 @@ module Releases def external? !internal? end + + def hook_attrs + { + id: id, + external: external?, + link_type: link_type, + name: name, + url: url + } + end end end diff --git a/app/models/releases/source.rb b/app/models/releases/source.rb index 2f00d25d768..44760541290 100644 --- a/app/models/releases/source.rb +++ b/app/models/releases/source.rb @@ -24,6 +24,13 @@ module Releases format: format) end + def hook_attrs + { + format: format, + url: url + } + end + private def archive_prefix diff --git a/app/models/resource_timebox_event.rb b/app/models/resource_timebox_event.rb index dbb2b428c7b..ac164783945 100644 --- a/app/models/resource_timebox_event.rb +++ b/app/models/resource_timebox_event.rb @@ -29,10 +29,10 @@ class ResourceTimeboxEvent < ResourceEvent case self when ResourceMilestoneEvent Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_milestone_changed_action(author: user) - when ResourceIterationEvent - Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_iteration_changed_action(author: user) else # no-op end end end + +ResourceTimeboxEvent.prepend_if_ee('EE::ResourceTimeboxEvent') diff --git a/app/models/route.rb b/app/models/route.rb index 706589e79b8..fe4846b3be5 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -20,6 +20,7 @@ class Route < ApplicationRecord scope :inside_path, -> (path) { where('routes.path LIKE ?', "#{sanitize_sql_like(path)}/%") } scope :for_routable, -> (routable) { where(source: routable) } + scope :for_routable_type, -> (routable_type) { where(source_type: routable_type) } scope :sort_by_path_length, -> { order('LENGTH(routes.path)', :path) } def rename_descendants diff --git a/app/models/service.rb b/app/models/service.rb index 764f417362f..2b6971954e3 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -8,6 +8,7 @@ class Service < ApplicationRecord include ProjectServicesLoggable include DataFields include FromUnion + include EachBatch SERVICE_NAMES = %w[ alerts asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker discord @@ -16,6 +17,7 @@ class Service < ApplicationRecord pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack ].freeze + # Fake services to help with local development. DEV_SERVICE_NAMES = %w[ mock_ci mock_deployment mock_monitoring ].freeze @@ -64,9 +66,9 @@ class Service < ApplicationRecord scope :by_type, -> (type) { where(type: type) } scope :by_active_flag, -> (flag) { where(active: flag) } scope :inherit_from_id, -> (id) { where(inherit_from_id: id) } - scope :for_group, -> (group) { where(group_id: group, type: available_services_types) } - scope :for_template, -> { where(template: true, type: available_services_types) } - scope :for_instance, -> { where(instance: true, type: available_services_types) } + scope :for_group, -> (group) { where(group_id: group, type: available_services_types(include_project_specific: false)) } + scope :for_template, -> { where(template: true, type: available_services_types(include_project_specific: false)) } + scope :for_instance, -> { where(instance: true, type: available_services_types(include_project_specific: false)) } scope :push_hooks, -> { where(push_events: true, active: true) } scope :tag_push_hooks, -> { where(tag_push_events: true, active: true) } @@ -167,13 +169,13 @@ class Service < ApplicationRecord end private_class_method :create_nonexistent_templates - def self.find_or_initialize_integration(name, instance: false, group_id: nil) - if name.in?(available_services_names) + def self.find_or_initialize_non_project_specific_integration(name, instance: false, group_id: nil) + if name.in?(available_services_names(include_project_specific: false)) "#{name}_service".camelize.constantize.find_or_initialize_by(instance: instance, group_id: group_id) end end - def self.find_or_initialize_all(scope) + def self.find_or_initialize_all_non_project_specific(scope) scope + build_nonexistent_services_for(scope) end @@ -187,13 +189,14 @@ class Service < ApplicationRecord def self.list_nonexistent_services_for(scope) # Using #map instead of #pluck to save one query count. This is because # ActiveRecord loaded the object here, so we don't need to query again later. - available_services_types - scope.map(&:type) + available_services_types(include_project_specific: false) - scope.map(&:type) end private_class_method :list_nonexistent_services_for - def self.available_services_names + def self.available_services_names(include_project_specific: true, include_dev: true) service_names = services_names - service_names += dev_services_names + service_names += project_specific_services_names if include_project_specific + service_names += dev_services_names if include_dev service_names.sort_by(&:downcase) end @@ -212,12 +215,10 @@ class Service < ApplicationRecord [] end - def self.available_services_types - available_services_names.map { |service_name| "#{service_name}_service".camelize } - end - - def self.services_types - services_names.map { |service_name| "#{service_name}_service".camelize } + def self.available_services_types(include_project_specific: true, include_dev: true) + available_services_names(include_project_specific: include_project_specific, include_dev: include_dev).map do |service_name| + "#{service_name}_service".camelize + end end def self.build_from_integration(integration, project_id: nil, group_id: nil) @@ -273,6 +274,17 @@ class Service < ApplicationRecord end end + def self.inherited_descendants_from_self_or_ancestors_from(integration) + inherit_from_ids = + where(type: integration.type, group: integration.group.self_and_ancestors) + .or(where(type: integration.type, instance: true)).select(:id) + + from_union([ + where(type: integration.type, inherit_from_id: inherit_from_ids, group: integration.group.descendants), + where(type: integration.type, inherit_from_id: inherit_from_ids, project: Project.in_namespace(integration.group.self_and_descendants)) + ]) + end + def activated? active end @@ -294,7 +306,7 @@ class Service < ApplicationRecord end def initialize_properties - self.properties = {} if properties.nil? + self.properties = {} if has_attribute?(:properties) && properties.nil? end def title @@ -410,8 +422,12 @@ class Service < ApplicationRecord ProjectServiceWorker.perform_async(id, data) end - def issue_tracker? - self.category == :issue_tracker + def external_issue_tracker? + category == :issue_tracker && active? + end + + def external_wiki? + type == 'ExternalWikiService' && active? end # override if needed diff --git a/app/models/snippet.rb b/app/models/snippet.rb index d71853e11cf..dc370b46bda 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -293,9 +293,7 @@ class Snippet < ApplicationRecord @storage ||= Storage::Hashed.new(self, prefix: Storage::Hashed::SNIPPET_REPOSITORY_PATH_PREFIX) end - # This is the full_path used to identify the - # the snippet repository. It will be used mostly - # for logging purposes. + # This is the full_path used to identify the the snippet repository. override :full_path def full_path return unless persisted? @@ -303,7 +301,7 @@ class Snippet < ApplicationRecord @full_path ||= begin components = [] components << project.full_path if project_id? - components << '@snippets' + components << 'snippets' components << self.id components.join('/') end diff --git a/app/models/terraform/state.rb b/app/models/terraform/state.rb index 9d88db27449..d329b429c9d 100644 --- a/app/models/terraform/state.rb +++ b/app/models/terraform/state.rb @@ -17,8 +17,15 @@ module Terraform belongs_to :project belongs_to :locked_by_user, class_name: 'User' - has_many :versions, class_name: 'Terraform::StateVersion', foreign_key: :terraform_state_id - has_one :latest_version, -> { ordered_by_version_desc }, class_name: 'Terraform::StateVersion', foreign_key: :terraform_state_id + has_many :versions, + class_name: 'Terraform::StateVersion', + foreign_key: :terraform_state_id, + inverse_of: :terraform_state + + has_one :latest_version, -> { ordered_by_version_desc }, + class_name: 'Terraform::StateVersion', + foreign_key: :terraform_state_id, + inverse_of: :terraform_state scope :versioning_not_enabled, -> { where(versioning_enabled: false) } scope :ordered_by_name, -> { order(:name) } @@ -48,11 +55,11 @@ module Terraform self.lock_xid.present? end - def update_file!(data, version:) + def update_file!(data, version:, build:) if versioning_enabled? - create_new_version!(data: data, version: version) + create_new_version!(data: data, version: version, build: build) elsif latest_version.present? - migrate_legacy_version!(data: data, version: version) + migrate_legacy_version!(data: data, version: version, build: build) else self.file = data save! @@ -81,18 +88,18 @@ module Terraform # The code can be removed in the next major version (14.0), after # which any states that haven't been migrated will need to be # recreated: https://gitlab.com/gitlab-org/gitlab/-/issues/258960 - def migrate_legacy_version!(data:, version:) + def migrate_legacy_version!(data:, version:, build:) current_file = latest_version.file.read current_version = parse_serial(current_file) || version - 1 update!(versioning_enabled: true) reload_latest_version.update!(version: current_version, file: CarrierWaveStringFile.new(current_file)) - create_new_version!(data: data, version: version) + create_new_version!(data: data, version: version, build: build) end - def create_new_version!(data:, version:) - new_version = versions.build(version: version, created_by_user: locked_by_user) + def create_new_version!(data:, version:, build:) + new_version = versions.build(version: version, created_by_user: locked_by_user, build: build) new_version.assign_attributes(file: data) new_version.save! end diff --git a/app/models/terraform/state_version.rb b/app/models/terraform/state_version.rb index eff44485401..cc5d94b8e09 100644 --- a/app/models/terraform/state_version.rb +++ b/app/models/terraform/state_version.rb @@ -6,6 +6,7 @@ module Terraform belongs_to :terraform_state, class_name: 'Terraform::State', optional: false belongs_to :created_by_user, class_name: 'User', optional: true + belongs_to :build, class_name: 'Ci::Build', optional: true, foreign_key: :ci_build_id scope :ordered_by_version_desc, -> { order(version: :desc) } diff --git a/app/models/user.rb b/app/models/user.rb index ef77e207215..be64e057d59 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -28,6 +28,8 @@ class User < ApplicationRecord DEFAULT_NOTIFICATION_LEVEL = :participating + INSTANCE_ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 + add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } add_authentication_token_field :feed_token add_authentication_token_field :static_object_token @@ -341,6 +343,7 @@ class User < ApplicationRecord # Scopes scope :admins, -> { where(admin: true) } + scope :instance_access_request_approvers_to_be_notified, -> { admins.active.order_recent_sign_in.limit(INSTANCE_ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :blocked_pending_approval, -> { with_states(:blocked_pending_approval) } scope :external, -> { where(external: true) } @@ -350,6 +353,9 @@ class User < ApplicationRecord scope :deactivated, -> { with_state(:deactivated).non_internal } scope :without_projects, -> { joins('LEFT JOIN project_authorizations ON users.id = project_authorizations.user_id').where(project_authorizations: { user_id: nil }) } scope :by_username, -> (usernames) { iwhere(username: Array(usernames).map(&:to_s)) } + scope :by_name, -> (names) { iwhere(name: Array(names)) } + scope :by_user_email, -> (emails) { iwhere(email: Array(emails)) } + scope :by_emails, -> (emails) { joins(:emails).where(emails: { email: Array(emails).map(&:downcase) }) } scope :for_todos, -> (todos) { where(id: todos.select(:user_id)) } scope :with_emails, -> { preload(:emails) } scope :with_dashboard, -> (dashboard) { where(dashboard: dashboard) } @@ -514,17 +520,15 @@ class User < ApplicationRecord # @param emails [String, Array<String>] email addresses to check # @param confirmed [Boolean] Only return users where the email is confirmed def by_any_email(emails, confirmed: false) - emails = Array(emails).map(&:downcase) - - from_users = where(email: emails) + from_users = by_user_email(emails) from_users = from_users.confirmed if confirmed - from_emails = joins(:emails).where(emails: { email: emails }) + from_emails = by_emails(emails) from_emails = from_emails.confirmed.merge(Email.confirmed) if confirmed items = [from_users, from_emails] - user_ids = Gitlab::PrivateCommitEmail.user_ids_for_emails(emails) + user_ids = Gitlab::PrivateCommitEmail.user_ids_for_emails(Array(emails).map(&:downcase)) items << where(id: user_ids) if user_ids.present? from_union(items) @@ -909,10 +913,11 @@ class User < ApplicationRecord # Returns the groups a user has access to, either through a membership or a project authorization def authorized_groups Group.unscoped do - Group.from_union([ - groups, - authorized_projects.joins(:namespace).select('namespaces.*') - ]) + if Feature.enabled?(:shared_group_membership_auth, self) + authorized_groups_with_shared_membership + else + authorized_groups_without_shared_membership + end end end @@ -1807,6 +1812,26 @@ class User < ApplicationRecord private + def authorized_groups_without_shared_membership + Group.from_union([ + groups, + authorized_projects.joins(:namespace).select('namespaces.*') + ]) + end + + def authorized_groups_with_shared_membership + cte = Gitlab::SQL::CTE.new(:direct_groups, authorized_groups_without_shared_membership) + cte_alias = cte.table.alias(Group.table_name) + + Group + .with(cte.to_arel) + .from_union([ + Group.from(cte_alias), + Group.joins(:shared_with_group_links) + .where(group_group_links: { shared_with_group_id: Group.from(cte_alias) }) + ]) + end + def default_private_profile_to_false return unless private_profile_changed? && private_profile.nil? @@ -1843,15 +1868,15 @@ class User < ApplicationRecord valid = true error = nil - if Gitlab::CurrentSettings.domain_blacklist_enabled? - blocked_domains = Gitlab::CurrentSettings.domain_blacklist + if Gitlab::CurrentSettings.domain_denylist_enabled? + blocked_domains = Gitlab::CurrentSettings.domain_denylist if domain_matches?(blocked_domains, email) error = 'is not from an allowed domain.' valid = false end end - allowed_domains = Gitlab::CurrentSettings.domain_whitelist + allowed_domains = Gitlab::CurrentSettings.domain_allowlist unless allowed_domains.blank? if domain_matches?(allowed_domains, email) valid = true diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index e39ff8712fc..cfad58fc0db 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -25,7 +25,8 @@ class UserCallout < ApplicationRecord personal_access_token_expiry: 21, # EE-only suggest_pipeline: 22, customize_homepage: 23, - feature_flags_new_version: 24 + feature_flags_new_version: 24, + registration_enabled_callout: 25 } validates :user, presence: true diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index c05bc80415a..b49a7eb72dc 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -1,11 +1,15 @@ # frozen_string_literal: true class UserPreference < ApplicationRecord + include IgnorableColumns + # We could use enums, but Rails 4 doesn't support multiple # enum options with same name for multiple fields, also it creates # extra methods that aren't really needed here. NOTES_FILTERS = { all_notes: 0, only_comments: 1, only_activity: 2 }.freeze + ignore_column :feature_filter_type, remove_with: '13.8', remove_after: '2021-01-22' + belongs_to :user scope :with_user, -> { joins(:user) } diff --git a/app/models/user_status.rb b/app/models/user_status.rb index 016b89bae81..0e1ae0b7338 100644 --- a/app/models/user_status.rb +++ b/app/models/user_status.rb @@ -9,6 +9,8 @@ class UserStatus < ApplicationRecord belongs_to :user + enum availability: { not_set: 0, busy: 1 } + validates :user, presence: true validates :emoji, inclusion: { in: Gitlab::Emoji.emojis_names } validates :message, length: { maximum: 100 }, allow_blank: true diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb index a4338c4e2bd..ab29afd0d08 100644 --- a/app/models/vulnerability.rb +++ b/app/models/vulnerability.rb @@ -2,6 +2,27 @@ # Placeholder class for model that is implemented in EE class Vulnerability < ApplicationRecord + include IgnorableColumns + + def self.link_reference_pattern + nil + end + + def self.reference_prefix + '[vulnerability:' + end + + def self.reference_prefix_escaped + '[vulnerability[' + end + + def self.reference_postfix + ']' + end + + def self.reference_postfix_escaped + ']' + end end Vulnerability.prepend_if_ee('EE::Vulnerability') |