diff options
Diffstat (limited to 'app/models')
122 files changed, 1585 insertions, 462 deletions
diff --git a/app/models/analytics/cycle_analytics/project_stage.rb b/app/models/analytics/cycle_analytics/project_stage.rb index a312bd24e78..23f0db0829b 100644 --- a/app/models/analytics/cycle_analytics/project_stage.rb +++ b/app/models/analytics/cycle_analytics/project_stage.rb @@ -9,6 +9,7 @@ module Analytics belongs_to :project alias_attribute :parent, :project + alias_attribute :parent_id, :project_id end end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 92526def144..a07933d4975 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -210,6 +210,16 @@ class ApplicationSetting < ApplicationRecord presence: true, if: :static_objects_external_storage_url? + validates :protected_paths, + length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, + allow_nil: false + + validates :push_event_hooks_limit, + numericality: { greater_than_or_equal_to: 0 } + + validates :push_event_activities_limit, + numericality: { greater_than_or_equal_to: 0 } + SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 8d9597aa5a4..0c0ffb67c9a 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -4,7 +4,7 @@ module ApplicationSettingImplementation extend ActiveSupport::Concern include Gitlab::Utils::StrongMemoize - DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace + STRING_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace | # or \s # any whitespace character | # or @@ -16,6 +16,19 @@ module ApplicationSettingImplementation FORBIDDEN_KEY_VALUE = KeyRestrictionValidator::FORBIDDEN SUPPORTED_KEY_TYPES = %i[rsa dsa ecdsa ed25519].freeze + DEFAULT_PROTECTED_PATHS = [ + '/users/password', + '/users/sign_in', + '/api/v3/session.json', + '/api/v3/session', + '/api/v4/session.json', + '/api/v4/session', + '/users', + '/users/confirmation', + '/unsubscribes/', + '/import/github/personal_access_token' + ].freeze + class_methods do def defaults { @@ -69,6 +82,8 @@ module ApplicationSettingImplementation polling_interval_multiplier: 1, project_export_enabled: true, protected_ci_variables: false, + push_event_hooks_limit: 3, + push_event_activities_limit: 3, raw_blob_request_limit: 300, recaptcha_enabled: false, login_recaptcha_protection_enabled: false, @@ -92,6 +107,13 @@ module ApplicationSettingImplementation throttle_unauthenticated_enabled: false, throttle_unauthenticated_period_in_seconds: 3600, throttle_unauthenticated_requests_per_period: 3600, + throttle_protected_paths_enabled: false, + throttle_protected_paths_in_seconds: 10, + throttle_protected_paths_per_period: 60, + protected_paths: DEFAULT_PROTECTED_PATHS, + throttle_incident_management_notification_enabled: false, + throttle_incident_management_notification_period_in_seconds: 3600, + throttle_incident_management_notification_per_period: 3600, time_tracking_limit_to_hours: false, two_factor_grace_period: 48, unique_ips_limit_enabled: false, @@ -106,7 +128,8 @@ module ApplicationSettingImplementation snowplow_collector_hostname: nil, snowplow_cookie_domain: nil, snowplow_enabled: false, - snowplow_site_id: nil + snowplow_site_id: nil, + custom_http_clone_url_root: nil } end @@ -149,11 +172,11 @@ module ApplicationSettingImplementation end def domain_whitelist_raw=(values) - self.domain_whitelist = domain_strings_to_array(values) + self.domain_whitelist = strings_to_array(values) end def domain_blacklist_raw=(values) - self.domain_blacklist = domain_strings_to_array(values) + self.domain_blacklist = strings_to_array(values) end def domain_blacklist_file=(file) @@ -167,7 +190,7 @@ module ApplicationSettingImplementation def outbound_local_requests_whitelist_raw=(values) clear_memoization(:outbound_local_requests_whitelist_arrays) - self.outbound_local_requests_whitelist = domain_strings_to_array(values) + self.outbound_local_requests_whitelist = strings_to_array(values) end def add_to_outbound_local_requests_whitelist(values_array) @@ -200,8 +223,16 @@ module ApplicationSettingImplementation end end + def protected_paths_raw + array_to_string(self.protected_paths) + end + + def protected_paths_raw=(values) + self.protected_paths = strings_to_array(values) + end + def asset_proxy_whitelist=(values) - values = domain_strings_to_array(values) if values.is_a?(String) + values = strings_to_array(values) if values.is_a?(String) # make sure we always whitelist the running host values << Gitlab.config.gitlab.host unless values.include?(Gitlab.config.gitlab.host) @@ -316,11 +347,11 @@ module ApplicationSettingImplementation arr&.join("\n") end - def domain_strings_to_array(values) + def strings_to_array(values) return [] unless values values - .split(DOMAIN_LIST_SEPARATOR) + .split(STRING_LIST_SEPARATOR) .map(&:strip) .reject(&:empty?) .uniq diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index c2eef500fb0..06a607b75a4 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class AuditEvent < ApplicationRecord + include CreatedAtFilterable + serialize :details, Hash # rubocop:disable Cop/ActiveRecordSerialize belongs_to :user, foreign_key: :author_id @@ -9,6 +11,9 @@ class AuditEvent < ApplicationRecord validates :entity_id, presence: true validates :entity_type, presence: true + scope :by_entity_type, -> (entity_type) { where(entity_type: entity_type) } + scope :by_entity_id, -> (entity_id) { where(entity_id: entity_id) } + after_initialize :initialize_details def initialize_details @@ -18,6 +23,10 @@ class AuditEvent < ApplicationRecord def author_name self.user.name end + + def formatted_details + details.merge(details.slice(:from, :to).transform_values(&:to_s)) + end end AuditEvent.prepend_if_ee('EE::AuditEvent') diff --git a/app/models/aws/role.rb b/app/models/aws/role.rb new file mode 100644 index 00000000000..836107435ad --- /dev/null +++ b/app/models/aws/role.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Aws + class Role < ApplicationRecord + self.table_name = 'aws_roles' + + belongs_to :user, inverse_of: :aws_role + + validates :role_external_id, uniqueness: true, length: { in: 1..64 } + validates :role_arn, + length: 1..2048, + format: { + with: Gitlab::Regex.aws_arn_regex, + message: Gitlab::Regex.aws_arn_regex_message + } + end +end diff --git a/app/models/blob.rb b/app/models/blob.rb index a590536d5fe..cc089715b06 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -32,6 +32,7 @@ class Blob < SimpleDelegator BlobViewer::Balsamiq, BlobViewer::Video, + BlobViewer::Audio, BlobViewer::PDF, @@ -176,7 +177,11 @@ class Blob < SimpleDelegator end def video? - UploaderHelper::VIDEO_EXT.include?(extension) + UploaderHelper::SAFE_VIDEO_EXT.include?(extension) + end + + def audio? + UploaderHelper::SAFE_AUDIO_EXT.include?(extension) end def readable_text? diff --git a/app/models/blob_viewer/audio.rb b/app/models/blob_viewer/audio.rb new file mode 100644 index 00000000000..cc7fe3b0d90 --- /dev/null +++ b/app/models/blob_viewer/audio.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module BlobViewer + class Audio < Base + include Rich + include ClientSide + + self.partial_name = 'audio' + self.extensions = UploaderHelper::SAFE_AUDIO_EXT + self.binary = true + end +end diff --git a/app/models/blob_viewer/image.rb b/app/models/blob_viewer/image.rb index 56e27839fca..cbebef46c60 100644 --- a/app/models/blob_viewer/image.rb +++ b/app/models/blob_viewer/image.rb @@ -6,7 +6,7 @@ module BlobViewer include ClientSide self.partial_name = 'image' - self.extensions = UploaderHelper::IMAGE_EXT + self.extensions = UploaderHelper::SAFE_IMAGE_EXT self.binary = true self.switcher_icon = 'picture-o' self.switcher_title = 'image' diff --git a/app/models/blob_viewer/video.rb b/app/models/blob_viewer/video.rb index 48bb2a13518..3ec4e90b24e 100644 --- a/app/models/blob_viewer/video.rb +++ b/app/models/blob_viewer/video.rb @@ -6,9 +6,7 @@ module BlobViewer include ClientSide self.partial_name = 'video' - self.extensions = UploaderHelper::VIDEO_EXT + self.extensions = UploaderHelper::SAFE_VIDEO_EXT self.binary = true - self.switcher_icon = 'film' - self.switcher_title = 'video' end end diff --git a/app/models/board.rb b/app/models/board.rb index 31011dc4742..f3f938224a4 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -16,10 +16,9 @@ class Board < ApplicationRecord !group end - def parent - @parent ||= group || project + def resource_parent + @resource_parent ||= group || project end - alias_method :resource_parent, :parent def group_board? group_id.present? diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb index ef00ad75683..76d4b9d6206 100644 --- a/app/models/ci/artifact_blob.rb +++ b/app/models/ci/artifact_blob.rb @@ -53,7 +53,7 @@ module Ci pages_config.enabled && pages_config.artifacts_server && EXTENSIONS_SERVED_BY_PAGES.include?(File.extname(name)) && - job.project.public? + (pages_config.access_control || job.project.public?) end private diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1f8a0373450..c48ab28ce73 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -12,7 +12,6 @@ module Ci include Presentable include Importable include Gitlab::Utils::StrongMemoize - include Deployable include HasRef BuildArchivedError = Class.new(StandardError) @@ -43,6 +42,7 @@ module Ci has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id Ci::JobArtifact.file_types.each do |key, value| has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id @@ -118,8 +118,6 @@ module Ci scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } - scope :with_artifacts_stored_locally, -> { with_existing_job_artifacts(Ci::JobArtifact.archive.with_files_stored_locally) } - scope :with_archived_trace_stored_locally, -> { with_existing_job_artifacts(Ci::JobArtifact.trace.with_files_stored_locally) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } @@ -130,6 +128,12 @@ module Ci scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) } scope :finished_before, -> (date) { finished.where('finished_at < ?', date) } + scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") } + + scope :with_secure_reports_from_config_options, -> (job_types) do + joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) + end + scope :matches_tag_ids, -> (tag_ids) do matcher = ::ActsAsTaggableOn::Tagging .where(taggable_type: CommitStatus.name) @@ -236,6 +240,7 @@ module Ci end after_transition pending: :running do |build| + build.pipeline.persistent_ref.create build.deployment&.run build.run_after_commit do @@ -412,10 +417,6 @@ module Ci self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options end - def has_deployment? - !!self.deployment - end - def outdated_deployment? success? && !deployment.try(:last?) end @@ -753,6 +754,10 @@ module Ci true end + def invalid_dependencies + dependencies.reject(&:valid_dependency?) + end + def runner_required_feature_names strong_memoize(:runner_required_feature_names) do RUNNER_FEATURES.select do |feature, method| diff --git a/app/models/ci/build_trace.rb b/app/models/ci/build_trace.rb new file mode 100644 index 00000000000..b9db1559836 --- /dev/null +++ b/app/models/ci/build_trace.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Ci + class BuildTrace + CONVERTERS = { + html: Gitlab::Ci::Ansi2html, + json: Gitlab::Ci::Ansi2json + }.freeze + + attr_reader :trace, :build + + delegate :state, :append, :truncated, :offset, :size, :total, to: :trace, allow_nil: true + delegate :id, :status, :complete?, to: :build, prefix: true + + def initialize(build:, stream:, state:, content_format:) + @build = build + @content_format = content_format + + if stream.valid? + stream.limit + @trace = CONVERTERS.fetch(content_format).convert(stream.stream, state) + end + end + + def json? + @content_format == :json + end + + def html? + @content_format == :html + end + + def json_lines + @trace&.lines if json? + end + + def html_lines + @trace&.html if html? + end + end +end diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb index 8be42eb48d6..7fe6b753da1 100644 --- a/app/models/ci/build_trace_section.rb +++ b/app/models/ci/build_trace_section.rb @@ -4,6 +4,9 @@ module Ci class BuildTraceSection < ApplicationRecord extend Gitlab::Ci::Model + # Only remove > 2019-11-22 and > 12.5 + self.ignored_columns += %i[id] + belongs_to :build, class_name: 'Ci::Build' belongs_to :project belongs_to :section_name, class_name: 'Ci::BuildTraceSectionName' diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 9b2c3c807ac..0e05318b253 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -9,6 +9,7 @@ module Ci # class Group include StaticModel + include Gitlab::Utils::StrongMemoize attr_reader :stage, :name, :jobs @@ -21,7 +22,17 @@ module Ci end def status - @status ||= commit_statuses.status + strong_memoize(:status) do + if Feature.enabled?(:ci_composite_status, default_enabled: false) + Gitlab::Ci::Status::Composite + .new(@jobs) + .status + else + CommitStatus + .where(id: @jobs) + .legacy_status + end + end end def detailed_status(current_user) @@ -40,11 +51,5 @@ module Ci self.new(stage, name: group_name, jobs: grouped_statuses) end end - - private - - def commit_statuses - @commit_statuses ||= CommitStatus.where(id: jobs.map(&:id)) - end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index da2758507ce..62bf2c3ac9c 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -5,6 +5,7 @@ module Ci include AfterCommitQueue include ObjectStorage::BackgroundMove include UpdateProjectStatistics + include Sortable extend Gitlab::Ci::Model NotSupportedAdapterError = Class.new(StandardError) @@ -64,6 +65,7 @@ module Ci after_save :update_file_store, if: :saved_change_to_file? scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } + scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :with_file_types, -> (file_types) do types = self.file_types.select { |file_type| file_types.include?(file_type) }.values @@ -143,6 +145,10 @@ module Ci self.update_column(:file_store, file.object_store) end + def self.total_size + self.sum(:size) + end + def self.artifacts_size_for(project) self.where(project: project).sum(:size) end diff --git a/app/models/ci/legacy_stage.rb b/app/models/ci/legacy_stage.rb index 930c8a71453..2fd369c9aff 100644 --- a/app/models/ci/legacy_stage.rb +++ b/app/models/ci/legacy_stage.rb @@ -14,7 +14,8 @@ module Ci @pipeline = pipeline @name = name @status = status - @warnings = warnings + # support ints and booleans + @has_warnings = ActiveRecord::Type::Boolean.new.cast(warnings) end def groups @@ -30,7 +31,7 @@ module Ci end def status - @status ||= statuses.latest.status + @status ||= statuses.latest.slow_composite_status end def detailed_status(current_user) @@ -52,11 +53,12 @@ module Ci end def has_warnings? - if @warnings.is_a?(Integer) - @warnings > 0 - else - statuses.latest.failed_but_allowed.any? + # lazilly calculate the warnings + if @has_warnings.nil? + @has_warnings = statuses.latest.failed_but_allowed.any? end + + @has_warnings end def manual_playable? diff --git a/app/models/ci/persistent_ref.rb b/app/models/ci/persistent_ref.rb new file mode 100644 index 00000000000..be3d4aa3203 --- /dev/null +++ b/app/models/ci/persistent_ref.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Ci + ## + # The persistent pipeline ref to ensure runners can safely fetch source code + # even if force-push/source-branch-deletion happens. + class PersistentRef + include ActiveModel::Model + + attr_accessor :pipeline + + delegate :project, :sha, to: :pipeline + delegate :repository, to: :project + delegate :ref_exists?, :create_ref, :delete_refs, to: :repository + + def exist? + ref_exists?(path) + rescue + false + end + + def create + return if exist? + + create_ref(sha, path) + rescue => e + Gitlab::Sentry + .track_acceptable_exception(e, extra: { pipeline_id: pipeline.id }) + end + + def delete + delete_refs(path) + rescue Gitlab::Git::Repository::NoRepository + # no-op + rescue => e + Gitlab::Sentry + .track_acceptable_exception(e, extra: { pipeline_id: pipeline.id }) + end + + def path + "refs/#{Repository::REF_PIPELINES}/#{pipeline.id}" + end + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 20b8be4017e..3bf19399cec 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -25,7 +25,7 @@ module Ci belongs_to :merge_request, class_name: 'MergeRequest' belongs_to :external_pull_request - has_internal_id :iid, scope: :project, presence: false, init: ->(s) do + has_internal_id :iid, scope: :project, presence: false, ensure_if: -> { !importing? }, init: ->(s) do s&.project&.all_pipelines&.maximum(:iid) || s&.project&.all_pipelines&.count end @@ -52,9 +52,15 @@ module Ci has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_pipeline_id + has_one :source_pipeline, class_name: 'Ci::Sources::Pipeline', inverse_of: :pipeline has_one :chat_data, class_name: 'Ci::PipelineChatData' + has_many :triggered_pipelines, through: :sourced_pipelines, source: :pipeline + has_one :triggered_by_pipeline, through: :source_pipeline, source: :source_pipeline + has_one :source_job, through: :source_pipeline, source: :source_job + accepts_nested_attributes_for :variables, reject_if: :persisted? delegate :id, to: :project, prefix: true @@ -174,6 +180,8 @@ module Ci after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline| pipeline.run_after_commit do + pipeline.persistent_ref.delete + pipeline.all_merge_requests.each do |merge_request| next unless merge_request.auto_merge_enabled? @@ -209,6 +217,8 @@ module Ci scope :for_sha, -> (sha) { where(sha: sha) } scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) } scope :for_sha_or_source_sha, -> (sha) { for_sha(sha).or(for_source_sha(sha)) } + scope :for_ref, -> (ref) { where(ref: ref) } + scope :for_id, -> (id) { where(id: id) } scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :triggered_by_merge_request, -> (merge_request) do @@ -279,16 +289,16 @@ module Ci end end - # Returns a Hash containing the latest pipeline status for every given + # Returns a Hash containing the latest pipeline for every given # commit. # - # The keys of this Hash are the commit SHAs, the values the statuses. + # The keys of this Hash are the commit SHAs, the values the pipelines. # - # commits - The list of commit SHAs to get the status for. + # commits - The list of commit SHAs to get the pipelines for. # ref - The ref to scope the data to (e.g. "master"). If the ref is not - # given we simply get the latest status for the commits, regardless - # of what refs their pipelines belong to. - def self.latest_status_per_commit(commits, ref = nil) + # given we simply get the latest pipelines for the commits, regardless + # of what refs the pipelines belong to. + def self.latest_pipeline_per_commit(commits, ref = nil) p1 = arel_table p2 = arel_table.alias @@ -302,15 +312,14 @@ module Ci cond = cond.and(p1[:ref].eq(p2[:ref])) if ref join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond) - relation = select(:sha, :status) - .where(sha: commits) + relation = where(sha: commits) .where(p2[:id].eq(nil)) .joins(join.join_sources) relation = relation.where(ref: ref) if ref - relation.each_with_object({}) do |row, hash| - hash[row[:sha]] = row[:status] + relation.each_with_object({}) do |pipeline, hash| + hash[pipeline.sha] = pipeline end end @@ -385,13 +394,12 @@ module Ci end end - def legacy_stages + def legacy_stages_using_sql # TODO, this needs refactoring, see gitlab-foss#26481. - stages_query = statuses .group('stage').select(:stage).order('max(stage_idx)') - status_sql = statuses.latest.where('stage=sg.stage').status_sql + status_sql = statuses.latest.where('stage=sg.stage').legacy_status_sql warnings_sql = statuses.latest.select('COUNT(*)') .where('stage=sg.stage').failed_but_allowed.to_sql @@ -404,6 +412,30 @@ module Ci end end + def legacy_stages_using_composite_status + stages = statuses.latest + .order(:stage_idx, :stage) + .group_by(&:stage) + + stages.map do |stage_name, jobs| + composite_status = Gitlab::Ci::Status::Composite + .new(jobs) + + Ci::LegacyStage.new(self, + name: stage_name, + status: composite_status.status, + warnings: composite_status.warnings?) + end + end + + def legacy_stages + if Feature.enabled?(:ci_composite_status, default_enabled: false) + legacy_stages_using_composite_status + else + legacy_stages_using_sql + end + end + def valid_commit_sha if self.sha == Gitlab::Git::BLANK_SHA self.errors.add(:sha, " cant be 00000000 (branch removal)") @@ -584,11 +616,7 @@ module Ci def ci_yaml_file_path return unless repository_source? || unknown_source? - if project.ci_config_path.blank? - '.gitlab-ci.yml' - else - project.ci_config_path - end + project.ci_config_path.presence || '.gitlab-ci.yml' end def ci_yaml_file @@ -638,7 +666,8 @@ module Ci def update_status retry_optimistic_lock(self) do - case latest_builds_status.to_s + new_status = latest_builds_status.to_s + case new_status when 'created' then nil when 'preparing' then prepare when 'pending' then enqueue @@ -651,7 +680,7 @@ module Ci when 'scheduled' then delay else raise HasStatus::UnknownStatusError, - "Unknown status `#{latest_builds_status}`" + "Unknown status `#{new_status}`" end end end @@ -725,6 +754,10 @@ module Ci end end + def all_merge_requests_by_recency + all_merge_requests.order(id: :desc) + end + def detailed_status(current_user) Gitlab::Ci::Status::Pipeline::Factory .new(self, current_user) @@ -771,6 +804,18 @@ module Ci end end + def all_worktree_paths + strong_memoize(:all_worktree_paths) do + project.repository.ls_files(sha) + end + end + + def top_level_worktree_paths + strong_memoize(:top_level_worktree_paths) do + project.repository.tree(sha).blobs.map(&:path) + end + end + def default_branch? ref == project.default_branch end @@ -845,6 +890,10 @@ module Ci end end + def persistent_ref + @persistent_ref ||= PersistentRef.new(pipeline: self) + end + private def ci_yaml_from_repo @@ -894,7 +943,7 @@ module Ci def latest_builds_status return 'failed' unless yaml_errors.blank? - statuses.latest.status || 'skipped' + statuses.latest.slow_composite_status || 'skipped' end def keep_around_commits diff --git a/app/models/ci/pipeline_enums.rb b/app/models/ci/pipeline_enums.rb index cb92aef4bda..859abc4a0d5 100644 --- a/app/models/ci/pipeline_enums.rb +++ b/app/models/ci/pipeline_enums.rb @@ -22,6 +22,7 @@ module Ci schedule: 4, api: 5, external: 6, + pipeline: 7, chat: 8, merge_request_event: 10, external_pull_request_event: 11 diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 42d4e86fe8d..946241b7d4c 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -86,3 +86,5 @@ module Ci end end end + +Ci::PipelineSchedule.prepend_if_ee('EE::Ci::PipelineSchedule') diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb new file mode 100644 index 00000000000..feaec27281c --- /dev/null +++ b/app/models/ci/sources/pipeline.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Ci + module Sources + class Pipeline < ApplicationRecord + self.table_name = "ci_sources_pipelines" + + belongs_to :project, class_name: "Project" + belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :source_pipeline + + belongs_to :source_project, class_name: "Project", foreign_key: :source_project_id + belongs_to :source_job, class_name: "CommitStatus", foreign_key: :source_job_id + belongs_to :source_pipeline, class_name: "Ci::Pipeline", foreign_key: :source_pipeline_id + + validates :project, presence: true + validates :pipeline, presence: true + + validates :source_project, presence: true + validates :source_job, presence: true + validates :source_pipeline, presence: true + end + end +end + +::Ci::Sources::Pipeline.prepend_if_ee('::EE::Ci::Sources::Pipeline') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index d90339d90dc..77ac8bfe875 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -78,7 +78,8 @@ module Ci def update_status retry_optimistic_lock(self) do - case statuses.latest.status + new_status = latest_stage_status.to_s + case new_status when 'created' then nil when 'preparing' then prepare when 'pending' then enqueue @@ -91,7 +92,7 @@ module Ci when 'skipped', nil then skip else raise HasStatus::UnknownStatusError, - "Unknown status `#{statuses.latest.status}`" + "Unknown status `#{new_status}`" end end end @@ -124,5 +125,9 @@ module Ci def manual_playable? blocked? || skipped? end + + def latest_stage_status + statuses.latest.slow_composite_status || 'skipped' + end end end diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index 8792c5cf98b..68548bd2fdc 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -45,3 +45,5 @@ module Ci end end end + +Ci::Trigger.prepend_if_ee('EE::Ci::Trigger') diff --git a/app/models/clusters/applications/cert_manager.rb b/app/models/clusters/applications/cert_manager.rb index 27d4180e5b9..18cbf827a67 100644 --- a/app/models/clusters/applications/cert_manager.rb +++ b/app/models/clusters/applications/cert_manager.rb @@ -65,7 +65,7 @@ module Clusters end def retry_command(command) - "for i in $(seq 1 30); do #{command} && break; sleep 1s; echo \"Retrying ($i)...\"; done" + "for i in $(seq 1 30); do #{command} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)" end def post_delete_script diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb index 455cf200fbc..4a1bcac4bb7 100644 --- a/app/models/clusters/applications/helm.rb +++ b/app/models/clusters/applications/helm.rb @@ -27,7 +27,7 @@ module Clusters def set_initial_status return unless not_installable? - self.status = 'installable' if cluster&.platform_kubernetes_active? + self.status = status_states[:installable] if cluster&.platform_kubernetes_active? end # It can only be uninstalled if there are no other applications installed @@ -68,6 +68,13 @@ module Clusters ca_key.present? && ca_cert.present? end + def post_uninstall + cluster.kubeclient.delete_namespace(Gitlab::Kubernetes::Helm::NAMESPACE) + rescue Kubeclient::ResourceNotFoundError + # we actually don't care if the namespace is not present + # since we want to delete it anyway. + end + private def files diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 44c66f06059..885e4ff7197 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Ingress < ApplicationRecord - VERSION = '1.1.2' + VERSION = '1.22.1' self.table_name = 'clusters_applications_ingress' diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index ec65482a846..ca93bc15be0 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -23,7 +23,7 @@ module Clusters return unless cluster&.application_ingress_available? ingress = cluster.application_ingress - self.status = 'installable' if ingress.external_ip_or_hostname? + self.status = status_states[:installable] if ingress.external_ip_or_hostname? end def chart diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb index a9b9374622d..1093efee85a 100644 --- a/app/models/clusters/applications/knative.rb +++ b/app/models/clusters/applications/knative.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Knative < ApplicationRecord - VERSION = '0.6.0' + VERSION = '0.7.0' REPOSITORY = 'https://storage.googleapis.com/triggermesh-charts' METRICS_CONFIG = 'https://storage.googleapis.com/triggermesh-charts/istio-metrics.yaml' FETCH_IP_ADDRESS_DELAY = 30.seconds @@ -21,7 +21,7 @@ module Clusters return unless not_installable? return unless verify_cluster? - self.status = 'installable' + self.status = status_states[:installable] end state_machine :status do @@ -47,6 +47,10 @@ module Clusters { "domain" => hostname }.to_yaml end + def allowed_to_uninstall? + !pre_installed? + end + def install_command Gitlab::Kubernetes::Helm::InstallCommand.new( name: name, diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 7a414d1a5bb..5e7fdd55cb6 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -112,7 +112,12 @@ module Clusters def delete_knative_istio_metrics return [] unless cluster.application_knative_available? - [Gitlab::Kubernetes::KubectlCmd.delete("-f", Clusters::Applications::Knative::METRICS_CONFIG)] + [ + Gitlab::Kubernetes::KubectlCmd.delete( + "-f", Clusters::Applications::Knative::METRICS_CONFIG, + "--ignore-not-found" + ) + ] end end end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 2d6af8f4f0b..954046c143b 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.8.0' + VERSION = '0.9.0' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 6a5b98a4676..d6f5d7c3f93 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -24,6 +24,7 @@ module Clusters KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN' belongs_to :user + belongs_to :management_project, class_name: '::Project', optional: true has_many :cluster_projects, class_name: 'Clusters::Project' has_many :projects, through: :cluster_projects, class_name: '::Project' @@ -34,12 +35,13 @@ module Clusters # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true + has_one :provider_aws, class_name: 'Clusters::Providers::Aws', autosave: true has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName application = APPLICATIONS[name.to_s] - has_one application.association_name, class_name: application.to_s # rubocop:disable Rails/ReflectionClassName + has_one application.association_name, class_name: application.to_s, inverse_of: :cluster # rubocop:disable Rails/ReflectionClassName end has_one_cluster_application :helm @@ -63,6 +65,7 @@ module Clusters validate :restrict_modification, on: :update validate :no_groups, unless: :group_type? validate :no_projects, unless: :project_type? + validate :unique_management_project_environment_scope after_save :clear_reactive_cache! @@ -94,14 +97,20 @@ module Clusters enum provider_type: { user: 0, - gcp: 1 + gcp: 1, + aws: 2 } scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } - scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) } - scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) } - scope :gcp_installed, -> { gcp_provided.includes(:provider_gcp).where(cluster_providers_gcp: { status: ::Clusters::Providers::Gcp.state_machines[:status].states[:created].value }) } + + scope :user_provided, -> { where(provider_type: :user) } + scope :gcp_provided, -> { where(provider_type: :gcp) } + scope :aws_provided, -> { where(provider_type: :aws) } + + scope :gcp_installed, -> { gcp_provided.joins(:provider_gcp).merge(Clusters::Providers::Gcp.with_status(:created)) } + scope :aws_installed, -> { aws_provided.joins(:provider_aws).merge(Clusters::Providers::Aws.with_status(:created)) } + scope :managed, -> { where(managed: true) } scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) } @@ -138,7 +147,11 @@ module Clusters end def provider - return provider_gcp if gcp? + if gcp? + provider_gcp + elsif aws? + provider_aws + end end def platform @@ -172,7 +185,7 @@ module Clusters persisted_namespace = Clusters::KubernetesNamespaceFinder.new( self, project: project, - environment_slug: environment.slug + environment_name: environment.name ).execute persisted_namespace&.namespace || Gitlab::Kubernetes::DefaultNamespace.new(self, project: project).from_environment_slug(environment.slug) @@ -194,8 +207,24 @@ module Clusters end end + def knative_pre_installed? + provider&.knative_pre_installed? + end + private + def unique_management_project_environment_scope + return unless management_project + + duplicate_management_clusters = management_project.management_clusters + .where(environment_scope: environment_scope) + .where.not(id: id) + + if duplicate_management_clusters.any? + errors.add(:environment_scope, "cannot add duplicated environment scope") + end + end + def instance_domain @instance_domain ||= Gitlab::CurrentSettings.auto_devops_domain end diff --git a/app/models/clusters/clusters_hierarchy.rb b/app/models/clusters/clusters_hierarchy.rb index 5556fc8d3f0..a906eb2888b 100644 --- a/app/models/clusters/clusters_hierarchy.rb +++ b/app/models/clusters/clusters_hierarchy.rb @@ -4,8 +4,9 @@ module Clusters class ClustersHierarchy DEPTH_COLUMN = :depth - def initialize(clusterable) + def initialize(clusterable, include_management_project: true) @clusterable = clusterable + @include_management_project = include_management_project end # Returns clusters in order from deepest to highest group @@ -24,7 +25,7 @@ module Clusters private - attr_reader :clusterable + attr_reader :clusterable, :include_management_project def recursive_cte cte = Gitlab::SQL::RecursiveCTE.new(:clusters_cte) @@ -38,12 +39,25 @@ module Clusters raise ArgumentError, "unknown type for #{clusterable}" end + if clusterable.is_a?(::Project) && include_management_project + cte << management_clusters_query + end + cte << base_query cte << parent_query(cte) cte end + # Management clusters should be first in the hierarchy so we use 0 for the + # depth column. + # + # group_parent_id is un-used but we still need to match the same number of + # columns as other queries in the CTE. + def management_clusters_query + clusterable.management_clusters.select([clusters_star, 'NULL AS group_parent_id', "0 AS #{DEPTH_COLUMN}"]) + end + def group_clusters_base_query group_parent_id_alias = alias_as_column(groups[:parent_id], 'group_parent_id') join_sources = ::Group.left_joins(:clusters).arel.join_sources diff --git a/app/models/clusters/concerns/application_core.rb b/app/models/clusters/concerns/application_core.rb index d1b57a21a7d..979cf0645f5 100644 --- a/app/models/clusters/concerns/application_core.rb +++ b/app/models/clusters/concerns/application_core.rb @@ -15,7 +15,7 @@ module Clusters def set_initial_status return unless not_installable? - self.status = 'installable' if cluster&.application_helm_available? + self.status = status_states[:installable] if cluster&.application_helm_available? end def can_uninstall? @@ -64,3 +64,5 @@ module Clusters end end end + +Clusters::Concerns::ApplicationCore.prepend_if_ee('EE::Clusters::Concerns::ApplicationCore') diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb index 342d766f723..b63a596dfee 100644 --- a/app/models/clusters/concerns/application_status.rb +++ b/app/models/clusters/concerns/application_status.rb @@ -28,6 +28,13 @@ module Clusters state :uninstalling, value: 7 state :uninstall_errored, value: 8 + # Used for applications that are pre-installed by the cluster, + # e.g. Knative in GCP Cloud Run enabled clusters + # Because we cannot upgrade or uninstall Knative in these clusters, + # we define only one simple state transition to enter the `pre_installed` state, + # and no exit transitions. + state :pre_installed, value: 9 + event :make_scheduled do transition [:installable, :errored, :installed, :updated, :update_errored, :uninstall_errored] => :scheduled end @@ -41,6 +48,10 @@ module Clusters transition [:updating] => :updated end + event :make_pre_installed do + transition any => :pre_installed + end + event :make_errored do transition any - [:updating, :uninstalling] => :errored transition [:updating] => :update_errored @@ -90,12 +101,18 @@ module Clusters end end + def status_states + self.class.state_machines[:status].states.each_with_object({}) do |state, states| + states[state.name] = state.value + end + end + def updateable? installed? || updated? || update_errored? end def available? - installed? || updated? + pre_installed? || installed? || updated? end def update_in_progress? diff --git a/app/models/clusters/concerns/application_version.rb b/app/models/clusters/concerns/application_version.rb index db94e8e08c9..6c0b014662c 100644 --- a/app/models/clusters/concerns/application_version.rb +++ b/app/models/clusters/concerns/application_version.rb @@ -8,13 +8,13 @@ module Clusters included do state_machine :status do before_transition any => [:installed, :updated] do |application| - application.version = application.class.const_get(:VERSION) + application.version = application.class.const_get(:VERSION, false) end end end def update_available? - version != self.class.const_get(:VERSION) + version != self.class.const_get(:VERSION, false) end end end diff --git a/app/models/clusters/concerns/provider_status.rb b/app/models/clusters/concerns/provider_status.rb new file mode 100644 index 00000000000..2da1ee7aabb --- /dev/null +++ b/app/models/clusters/concerns/provider_status.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Clusters + module Concerns + module ProviderStatus + extend ActiveSupport::Concern + + included do + state_machine :status, initial: :scheduled do + state :scheduled, value: 1 + state :creating, value: 2 + state :created, value: 3 + state :errored, value: 4 + + event :make_creating do + transition any - [:creating] => :creating + end + + event :make_created do + transition any - [:created] => :created + end + + event :make_errored do + transition any - [:errored] => :errored + end + + before_transition any => [:errored, :created] do |provider| + provider.nullify_credentials + end + + before_transition any => [:creating] do |provider, transition| + operation_id = transition.args.first + provider.assign_operation_id(operation_id) if operation_id + end + + before_transition any => [:errored] do |provider, transition| + status_reason = transition.args.first + provider.status_reason = status_reason if status_reason + end + end + + def on_creation? + scheduled? || creating? + end + + def assign_operation_id(_) + # Implemented by individual providers if operation ID is supported. + end + end + end + end +end diff --git a/app/models/clusters/kubernetes_namespace.rb b/app/models/clusters/kubernetes_namespace.rb index 69a2b99fcb6..42332bdc193 100644 --- a/app/models/clusters/kubernetes_namespace.rb +++ b/app/models/clusters/kubernetes_namespace.rb @@ -27,7 +27,7 @@ module Clusters algorithm: 'aes-256-cbc' scope :has_service_account_token, -> { where.not(encrypted_service_account_token: nil) } - scope :with_environment_slug, -> (slug) { joins(:environment).where(environments: { slug: slug }) } + scope :with_environment_name, -> (name) { joins(:environment).where(environments: { name: name }) } def token_name "#{namespace}-token" diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 89b50d8e8ff..314ef78757d 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -6,6 +6,7 @@ module Clusters include Gitlab::Kubernetes include EnumWithNil include AfterCommitQueue + include ReactiveCaching RESERVED_NAMESPACES = %w(gitlab-managed-apps).freeze @@ -23,11 +24,12 @@ module Clusters key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' + before_validation :nullify_blank_namespace before_validation :enforce_namespace_to_lower_case before_validation :enforce_ca_whitespace_trimming validates :namespace, - allow_blank: true, + allow_nil: true, length: 1..63, format: { with: Gitlab::Regex.kubernetes_namespace_regex, @@ -71,7 +73,7 @@ module Clusters .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true) end - if !cluster.managed? + if !cluster.managed? || cluster.management_project == project namespace = Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: project).from_environment_name(environment_name) variables @@ -105,19 +107,11 @@ module Clusters private - ## - # Environment slug can be predicted given an environment - # name, so even if the environment isn't persisted yet we - # still know what to look for. - def environment_slug(name) - Gitlab::Slug::Environment.new(name).generate - end - def find_persisted_namespace(project, environment_name:) Clusters::KubernetesNamespaceFinder.new( cluster, project: project, - environment_slug: environment_slug(environment_name) + environment_name: environment_name ).execute end @@ -198,6 +192,10 @@ module Clusters true end + + def nullify_blank_namespace + self.namespace = nil if namespace.blank? + end end end end diff --git a/app/models/clusters/providers/aws.rb b/app/models/clusters/providers/aws.rb new file mode 100644 index 00000000000..ae4156896bc --- /dev/null +++ b/app/models/clusters/providers/aws.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Clusters + module Providers + class Aws < ApplicationRecord + include Clusters::Concerns::ProviderStatus + + self.table_name = 'cluster_providers_aws' + + belongs_to :cluster, inverse_of: :provider_aws, class_name: 'Clusters::Cluster' + belongs_to :created_by_user, class_name: 'User' + + default_value_for :region, 'us-east-1' + default_value_for :num_nodes, 3 + default_value_for :instance_type, 'm5.large' + + attr_encrypted :secret_access_key, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_truncated, + algorithm: 'aes-256-gcm' + + validates :role_arn, + length: 1..2048, + format: { + with: Gitlab::Regex.aws_arn_regex, + message: Gitlab::Regex.aws_arn_regex_message + } + + validates :num_nodes, + numericality: { + only_integer: true, + greater_than: 0 + } + + validates :key_name, :region, :instance_type, :security_group_id, length: { in: 1..255 } + validates :subnet_ids, presence: true + + def nullify_credentials + assign_attributes( + access_key_id: nil, + secret_access_key: nil, + session_token: nil + ) + end + end + end +end diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index 390748bf252..f871674676f 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -3,6 +3,8 @@ module Clusters module Providers class Gcp < ApplicationRecord + include Clusters::Concerns::ProviderStatus + self.table_name = 'cluster_providers_gcp' belongs_to :cluster, inverse_of: :provider_gcp, class_name: 'Clusters::Cluster' @@ -10,6 +12,9 @@ module Clusters default_value_for :zone, 'us-central1-a' default_value_for :num_nodes, 3 default_value_for :machine_type, 'n1-standard-2' + default_value_for :cloud_run, false + + scope :cloud_run, -> { where(cloud_run: true) } attr_encrypted :access_token, mode: :per_attribute_iv, @@ -32,50 +37,25 @@ module Clusters greater_than: 0 } - state_machine :status, initial: :scheduled do - state :scheduled, value: 1 - state :creating, value: 2 - state :created, value: 3 - state :errored, value: 4 - - event :make_creating do - transition any - [:creating] => :creating - end - - event :make_created do - transition any - [:created] => :created - end - - event :make_errored do - transition any - [:errored] => :errored - end - - before_transition any => [:errored, :created] do |provider| - provider.access_token = nil - provider.operation_id = nil - end - - before_transition any => [:creating] do |provider, transition| - operation_id = transition.args.first - raise ArgumentError.new('operation_id is required') unless operation_id.present? - - provider.operation_id = operation_id - end + def api_client + return unless access_token - before_transition any => [:errored] do |provider, transition| - status_reason = transition.args.first - provider.status_reason = status_reason if status_reason - end + @api_client ||= GoogleApi::CloudPlatform::Client.new(access_token, nil) end - def on_creation? - scheduled? || creating? + def nullify_credentials + assign_attributes( + access_token: nil, + operation_id: nil + ) end - def api_client - return unless access_token + def assign_operation_id(operation_id) + assign_attributes(operation_id: operation_id) + end - @api_client ||= GoogleApi::CloudPlatform::Client.new(access_token, nil) + def knative_pre_installed? + cloud_run? end end end diff --git a/app/models/commit.rb b/app/models/commit.rb index a442f607fbf..aae49c36899 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -119,10 +119,22 @@ class Commit @raw = raw_commit @project = project - @statuses = {} @gpg_commit = Gitlab::Gpg::Commit.new(self) if project end + delegate \ + :pipelines, + :last_pipeline, + :latest_pipeline, + :latest_pipeline_for_project, + :set_latest_pipeline_for_ref, + :status, + to: :with_pipeline + + def with_pipeline + @with_pipeline ||= CommitWithPipeline.new(self) + end + def id raw.id end @@ -245,10 +257,9 @@ class Commit end def author - # We use __sync so that we get the actual objects back (including an actual - # nil), instead of a wrapper, as returning a wrapped nil breaks a lot of - # code. - lazy_author.__sync + strong_memoize(:author) do + lazy_author&.itself + end end request_cache(:author) { author_email.downcase } @@ -301,30 +312,6 @@ class Commit ) end - def pipelines - project.ci_pipelines.where(sha: sha) - end - - def last_pipeline - strong_memoize(:last_pipeline) do - pipelines.last - end - end - - def status(ref = nil) - return @statuses[ref] if @statuses.key?(ref) - - @statuses[ref] = status_for_project(ref, project) - end - - def status_for_project(ref, pipeline_project) - pipeline_project.ci_pipelines.latest_status_per_commit(id, ref)[id] - end - - def set_status_for_ref(ref, status) - @statuses[ref] = status - end - def signature return @signature if defined?(@signature) @@ -427,7 +414,7 @@ class Commit if entry[:type] == :blob blob = ::Blob.decorate(Gitlab::Git::Blob.new(name: entry[:name]), @project) - blob.image? || blob.video? ? :raw : :blob + blob.image? || blob.video? || blob.audio? ? :raw : :blob else entry[:type] end diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb index e8df46e1cc3..d4c29aa295b 100644 --- a/app/models/commit_collection.rb +++ b/app/models/commit_collection.rb @@ -34,6 +34,20 @@ class CommitCollection end end + # Returns the collection with the latest pipeline for every commit pre-set. + # + # Setting the pipeline for each commit ahead of time removes the need for running + # a query for every commit we're displaying. + def with_latest_pipeline(ref = nil) + pipelines = project.ci_pipelines.latest_pipeline_per_commit(map(&:id), ref) + + each do |commit| + commit.set_latest_pipeline_for_ref(ref, pipelines[commit.id]) + end + + self + end + def unenriched commits.reject(&:gitaly_commit?) end @@ -58,22 +72,15 @@ class CommitCollection end.compact] # Replace the commits, keeping the same order - @commits = @commits.map do |c| - replacements.fetch(c.id, c) - end - - self - end - - # Sets the pipeline status for every commit. - # - # Setting this status ahead of time removes the need for running a query for - # every commit we're displaying. - def with_pipeline_status - statuses = project.ci_pipelines.latest_status_per_commit(map(&:id), ref) - - each do |commit| - commit.set_status_for_ref(ref, statuses[commit.id]) + @commits = @commits.map do |original_commit| + # Return the original instance: if it didn't need to be batchloaded, it was + # already enriched. + batch_loaded_commit = replacements.fetch(original_commit.id, original_commit) + + # If batch loading the commit failed, fall back to the original commit. + # We need to explicitly check `.nil?` since otherwise a `BatchLoader` instance + # that looks like `nil` is returned. + batch_loaded_commit.nil? ? original_commit : batch_loaded_commit end self diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 5d9d3179f9d..39a6247b3b2 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -48,6 +48,10 @@ class CommitStatus < ApplicationRecord scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) } scope :for_ids, -> (ids) { where(id: ids) } + scope :with_preloads, -> do + preload(:project, :user) + end + scope :with_needs, -> (names = nil) do needs = Ci::BuildNeed.scoped_build.select(1) needs = needs.where(name: names) if names @@ -161,11 +165,11 @@ class CommitStatus < ApplicationRecord end def self.status_for_prior_stages(index) - before_stage(index).latest.status || 'success' + before_stage(index).latest.slow_composite_status || 'success' end def self.status_for_names(names) - where(name: names).latest.status || 'success' + where(name: names).latest.slow_composite_status || 'success' end def locking_enabled? diff --git a/app/models/commit_with_pipeline.rb b/app/models/commit_with_pipeline.rb new file mode 100644 index 00000000000..f382ae8f55a --- /dev/null +++ b/app/models/commit_with_pipeline.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class CommitWithPipeline < SimpleDelegator + include Presentable + + def initialize(commit) + @latest_pipelines = {} + super(commit) + end + + def pipelines + project.ci_pipelines.where(sha: sha) + end + + def last_pipeline + strong_memoize(:last_pipeline) do + pipelines.last + end + end + + def latest_pipeline(ref = nil) + @latest_pipelines.fetch(ref) do |ref| + @latest_pipelines[ref] = latest_pipeline_for_project(ref, project) + end + end + + def latest_pipeline_for_project(ref, pipeline_project) + pipeline_project.ci_pipelines.latest_pipeline_per_commit(id, ref)[id] + end + + def set_latest_pipeline_for_ref(ref, pipeline) + @latest_pipelines[ref] = pipeline + end + + def status(ref = nil) + latest_pipeline(ref)&.status + end +end diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb index 0c603c2d5e6..54e9a13d1ea 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage.rb @@ -7,6 +7,7 @@ module Analytics included do validates :name, presence: true + validates :name, exclusion: { in: Gitlab::Analytics::CycleAnalytics::DefaultStages.names }, if: :custom? validates :start_event_identifier, presence: true validates :end_event_identifier, presence: true validate :validate_stage_event_pairs @@ -15,6 +16,7 @@ module Analytics enum end_event_identifier: Gitlab::Analytics::CycleAnalytics::StageEvents.to_enum, _prefix: :end_event_identifier alias_attribute :custom_stage?, :custom + scope :default_stages, -> { where(custom: false) } end def parent=(_) @@ -45,11 +47,17 @@ module Analytics !custom end - # The model that is going to be queried, Issue or MergeRequest - def subject_model + # The model class that is going to be queried, Issue or MergeRequest + def subject_class start_event.object_type end + def matches_with_stage_params?(stage_params) + default_stage? && + start_event_identifier.to_s.eql?(stage_params[:start_event_identifier].to_s) && + end_event_identifier.to_s.eql?(stage_params[:end_event_identifier].to_s) + end + private def validate_stage_event_pairs diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index dc1735a7e48..64df265dc25 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -27,40 +27,73 @@ module AtomicInternalId extend ActiveSupport::Concern class_methods do - def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName + def has_internal_id(column, scope:, init:, ensure_if: nil, presence: true) # rubocop:disable Naming/PredicateName # We require init here to retain the ability to recalculate in the absence of a # InternaLId record (we may delete records in `internal_ids` for example). raise "has_internal_id requires a init block, none given." unless init + raise "has_internal_id needs to be defined on association." unless self.reflect_on_association(scope) - before_validation :"ensure_#{scope}_#{column}!", on: :create + before_validation :"track_#{scope}_#{column}!", on: :create + before_validation :"ensure_#{scope}_#{column}!", on: :create, if: ensure_if validates column, presence: presence define_method("ensure_#{scope}_#{column}!") do - scope_value = association(scope).reader + scope_value = internal_id_read_scope(scope) value = read_attribute(column) - return value unless scope_value - scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value } - usage = self.class.table_name.to_sym - - if value.present? - InternalId.track_greatest(self, scope_attrs, usage, value, init) - else - value = InternalId.generate_next(self, scope_attrs, usage, init) + if value.nil? + # We don't have a value yet and use a InternalId record to generate + # the next value. + value = InternalId.generate_next( + self, + internal_id_scope_attrs(scope), + internal_id_scope_usage, + init) write_attribute(column, value) end value end + define_method("track_#{scope}_#{column}!") do + return unless @internal_id_needs_tracking + + scope_value = internal_id_read_scope(scope) + return unless scope_value + + value = read_attribute(column) + + if value.present? + # The value was set externally, e.g. by the user + # We update the InternalId record to keep track of the greatest value. + InternalId.track_greatest( + self, + internal_id_scope_attrs(scope), + internal_id_scope_usage, + value, + init) + + @internal_id_needs_tracking = false + end + end + + define_method("#{column}=") do |value| + super(value).tap do |v| + # Indicate the iid was set from externally + @internal_id_needs_tracking = true + end + end + define_method("reset_#{scope}_#{column}") do if value = read_attribute(column) - scope_value = association(scope).reader - scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value } - usage = self.class.table_name.to_sym + did_reset = InternalId.reset( + self, + internal_id_scope_attrs(scope), + internal_id_scope_usage, + value) - if InternalId.reset(self, scope_attrs, usage, value) + if did_reset write_attribute(column, nil) end end @@ -69,4 +102,18 @@ module AtomicInternalId end end end + + def internal_id_scope_attrs(scope) + scope_value = internal_id_read_scope(scope) + + { scope_value.class.table_name.singularize.to_sym => scope_value } if scope_value + end + + def internal_id_scope_usage + self.class.table_name.to_sym + end + + def internal_id_read_scope(scope) + association(scope).reader + end end diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 269145309fc..a98baeb0e3d 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -38,7 +38,7 @@ module Avatarable def avatar_type unless self.avatar.image? - errors.add :avatar, "file format is not supported. Please try one of the following supported formats: #{AvatarUploader::IMAGE_EXT.join(', ')}" + errors.add :avatar, "file format is not supported. Please try one of the following supported formats: #{AvatarUploader::SAFE_IMAGE_EXT.join(', ')}" end end diff --git a/app/models/concerns/checksummable.rb b/app/models/concerns/checksummable.rb new file mode 100644 index 00000000000..1f76eb87aa5 --- /dev/null +++ b/app/models/concerns/checksummable.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Checksummable + extend ActiveSupport::Concern + + class_methods do + def hexdigest(path) + Digest::SHA256.file(path).hexdigest + end + end +end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index 91dda803031..49d6f3d399c 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -78,6 +78,7 @@ module Ci variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? variables.append(key: "CI_NODE_INDEX", value: self.options[:instance].to_s) if self.options&.include?(:instance) variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s) + variables.append(key: "CI_DEFAULT_BRANCH", value: project.default_branch) variables.concat(legacy_variables) end end diff --git a/app/models/concerns/ci/pipeline_delegator.rb b/app/models/concerns/ci/pipeline_delegator.rb index dbc5ed1bc9a..76e0cbc7dff 100644 --- a/app/models/concerns/ci/pipeline_delegator.rb +++ b/app/models/concerns/ci/pipeline_delegator.rb @@ -15,7 +15,8 @@ module Ci :merge_request_ref?, :source_ref, :source_ref_slug, - :legacy_detached_merge_request_pipeline?, to: :pipeline + :legacy_detached_merge_request_pipeline?, + :merge_train_pipeline?, to: :pipeline end end end diff --git a/app/models/concerns/deployable.rb b/app/models/concerns/deployable.rb deleted file mode 100644 index 957b72f3721..00000000000 --- a/app/models/concerns/deployable.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Deployable - extend ActiveSupport::Concern - - included do - after_create :create_deployment - - def create_deployment - return unless starts_environment? && !has_deployment? - - environment = project.environments.find_or_create_by( - name: expanded_environment_name - ) - - # If we failed to persist envirionment record by validation error, such as name with invalid character, - # the job will fall back to a non-environment job. - return unless environment.persisted? - - create_deployment!( - cluster_id: environment.deployment_platform&.cluster_id, - project_id: environment.project_id, - environment: environment, - ref: ref, - tag: tag, - sha: sha, - user: user, - on_stop: on_stop) - end - end -end diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index e1a8725e728..fe8e9609820 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -11,6 +11,10 @@ module DeploymentPlatform private + def cluster_management_project_enabled? + Feature.enabled?(:cluster_management_project, default_enabled: true) + end + def find_deployment_platform(environment) find_platform_kubernetes_with_cte(environment) || find_instance_cluster_platform_kubernetes(environment: environment) @@ -18,7 +22,7 @@ module DeploymentPlatform # EE would override this and utilize environment argument def find_platform_kubernetes_with_cte(_environment) - Clusters::ClustersHierarchy.new(self).base_and_ancestors + Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?).base_and_ancestors .enabled.default_environment .first&.platform_kubernetes end diff --git a/app/models/concerns/group_api_compatibility.rb b/app/models/concerns/group_api_compatibility.rb new file mode 100644 index 00000000000..f02aa2035e5 --- /dev/null +++ b/app/models/concerns/group_api_compatibility.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Add methods used by the groups API +module GroupAPICompatibility + extend ActiveSupport::Concern + + def project_creation_level_str + ::Gitlab::Access.project_creation_string_options.key(project_creation_level) + end + + def project_creation_level_str=(value) + write_attribute(:project_creation_level, ::Gitlab::Access.project_creation_string_options.fetch(value)) + end + + def subgroup_creation_level_str + ::Gitlab::Access.subgroup_creation_string_options.key(subgroup_creation_level) + end + + def subgroup_creation_level_str=(value) + write_attribute(:subgroup_creation_level, ::Gitlab::Access.subgroup_creation_string_options.fetch(value)) + end +end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index bcbbb27a9a8..c01fb4740e5 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -10,6 +10,8 @@ module HasStatus ACTIVE_STATUSES = %w[preparing pending running].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze ORDERED_STATUSES = %w[failed preparing pending running manual scheduled canceled success skipped created].freeze + PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze + EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7, scheduled: 8, preparing: 9 }.freeze @@ -17,7 +19,7 @@ module HasStatus UnknownStatusError = Class.new(StandardError) class_methods do - def status_sql + def legacy_status_sql scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all scope_warnings = respond_to?(:failed_but_allowed) ? failed_but_allowed : none @@ -53,8 +55,22 @@ module HasStatus ) end - def status - all.pluck(status_sql).first + def legacy_status + all.pluck(legacy_status_sql).first + end + + # This method should not be used. + # This method performs expensive calculation of status: + # 1. By plucking all related objects, + # 2. Or executes expensive SQL query + def slow_composite_status + if Feature.enabled?(:ci_composite_status, default_enabled: false) + Gitlab::Ci::Status::Composite + .new(all, with_allow_failure: columns_hash.key?('allow_failure')) + .status + else + legacy_status + end end def started_at diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index d02f3731cc2..852576dbbc2 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -4,7 +4,7 @@ # # Contains common functionality shared between Issues and MergeRequests # -# Used by Issue, MergeRequest +# Used by Issue, MergeRequest, Epic # module Issuable extend ActiveSupport::Concern @@ -25,6 +25,19 @@ module Issuable include UpdatedAtFilterable include IssuableStates include ClosedAtFilterable + include VersionedDescription + + TITLE_LENGTH_MAX = 255 + TITLE_HTML_LENGTH_MAX = 800 + DESCRIPTION_LENGTH_MAX = 1.megabyte + DESCRIPTION_HTML_LENGTH_MAX = 5.megabytes + + STATE_ID_MAP = { + opened: 1, + closed: 2, + merged: 3, + locked: 4 + }.with_indifferent_access.freeze # This object is used to gather issuable meta data for displaying # upvotes, downvotes, notes and closing merge requests count for issues and merge requests @@ -72,10 +85,15 @@ module Issuable prefix: true validates :author, presence: true - validates :title, presence: true, length: { maximum: 255 } - validates :description, length: { maximum: Gitlab::Database::MAX_TEXT_SIZE_LIMIT }, allow_blank: true + validates :title, presence: true, length: { maximum: TITLE_LENGTH_MAX } + # we validate the description against DESCRIPTION_LENGTH_MAX only for Issuables being created + # to avoid breaking the existing Issuables which may have their descriptions longer + validates :description, length: { maximum: DESCRIPTION_LENGTH_MAX }, allow_blank: true, on: :create + validate :description_max_length_for_new_records_is_valid, on: :update validate :milestone_is_valid + before_validation :truncate_description_on_import! + scope :authored, ->(user) { where(author_id: user) } scope :recent, -> { reorder(id: :desc) } scope :of_projects, ->(ids) { where(project_id: ids) } @@ -138,6 +156,16 @@ module Issuable def milestone_is_valid errors.add(:milestone_id, message: "is invalid") if milestone_id.present? && !milestone_available? end + + def description_max_length_for_new_records_is_valid + if new_record? && description.length > Issuable::DESCRIPTION_LENGTH_MAX + errors.add(:description, :too_long, count: Issuable::DESCRIPTION_LENGTH_MAX) + end + end + + def truncate_description_on_import! + self.description = description&.slice(0, Issuable::DESCRIPTION_LENGTH_MAX) if importing? + end end class_methods do @@ -152,13 +180,17 @@ module Issuable fuzzy_search(query, [:title]) end - # Available state values persisted in state_id column using state machine + def available_states + @available_states ||= STATE_ID_MAP.slice(*available_state_names) + end + + # Available state names used to persist state_id column using state machine # # Override this on subclasses if different states are needed # - # Check MergeRequest.available_states for example - def available_states - @available_states ||= { opened: 1, closed: 2 }.with_indifferent_access + # Check MergeRequest.available_states_names for example + def available_state_names + [:opened, :closed] end # Searches for records with a matching title or description. @@ -277,6 +309,14 @@ module Issuable end end + def state + self.class.available_states.key(state_id) + end + + def state=(value) + self.state_id = self.class.available_states[value] + end + def resource_parent project end diff --git a/app/models/concerns/issuable_states.rb b/app/models/concerns/issuable_states.rb index 33bc41d7f44..f0b9f0d1f3a 100644 --- a/app/models/concerns/issuable_states.rb +++ b/app/models/concerns/issuable_states.rb @@ -4,22 +4,20 @@ module IssuableStates extend ActiveSupport::Concern # The state:string column is being migrated to state_id:integer column - # This is a temporary hook to populate state_id column with new values - # and should be removed after the state column is removed. - # Check https://gitlab.com/gitlab-org/gitlab-foss/issues/51789 for more information + # This is a temporary hook to keep state column in sync until it is removed. + # Check https: https://gitlab.com/gitlab-org/gitlab/issues/33814 for more information + # The state column can be safely removed after 2019-10-27 included do - before_save :set_state_id + before_save :sync_issuable_deprecated_state end - def set_state_id - return if state.nil? || state.empty? + def sync_issuable_deprecated_state + return if self.is_a?(Epic) + return unless respond_to?(:state) + return if state_id.nil? - # Needed to prevent breaking some migration specs that - # rollback database to a point where state_id does not exist. - # We can use this guard clause for now since this file will - # be removed in the next release. - return unless self.has_attribute?(:state_id) + deprecated_state = self.class.available_states.key(state_id) - self.state_id = self.class.available_states[state] + self.write_attribute(:state, deprecated_state) end end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index 377600ef6e5..9b6c57261d8 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -150,7 +150,7 @@ module Mentionable # # Returns a Hash. def detect_mentionable_changes - source = (changes.present? ? changes : previous_changes).dup + source = (changes.presence || previous_changes).dup mentionable = self.class.mentionable_attrs.map { |attr, options| attr } diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index 3deb86da6cf..42b370990ac 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -6,7 +6,9 @@ module Milestoneish end def closed_issues_count(user) - count_issues_by_state(user)['closed'].to_i + closed_state_id = Issue.available_states[:closed] + + count_issues_by_state(user)[closed_state_id].to_i end def complete?(user) @@ -117,7 +119,7 @@ module Milestoneish def count_issues_by_state(user) memoize_per_user(user, :count_issues_by_state) do - issues_visible_to_user(user).reorder(nil).group(:state).count + issues_visible_to_user(user).reorder(nil).group(:state_id).count end end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index 6caa23ef9b7..3065e0ba6c5 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -7,6 +7,8 @@ module Noteable # avoiding n+1 queries and improving performance. NoteableMeta = Struct.new(:user_notes_count) + MAX_NOTES_LIMIT = 5_000 + class_methods do # `Noteable` class names that support replying to individual notes. def replyable_types diff --git a/app/models/concerns/notification_branch_selection.rb b/app/models/concerns/notification_branch_selection.rb index d8e18de7551..7f00b652530 100644 --- a/app/models/concerns/notification_branch_selection.rb +++ b/app/models/concerns/notification_branch_selection.rb @@ -21,7 +21,7 @@ module NotificationBranchSelection end is_default_branch = ref == project.default_branch - is_protected_branch = project.protected_branches.exists?(name: ref) + is_protected_branch = ProtectedBranch.protected?(project, ref) case branches_to_be_notified when "all" diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index aab0589f7ca..9df77b565da 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -44,7 +44,7 @@ module PrometheusAdapter end def query_klass_for(query_name) - Gitlab::Prometheus::Queries.const_get("#{query_name.to_s.classify}Query") + Gitlab::Prometheus::Queries.const_get("#{query_name.to_s.classify}Query", false) end def build_query_args(*args) diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index dfe3c391880..b645cf71443 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -127,6 +127,7 @@ module RelativePositioning if pos_after && (pos_after - pos_before) < 2 before.move_sequence_after + pos_after = before.next_relative_position end self.relative_position = self.class.position_between(pos_before, pos_after) @@ -138,6 +139,7 @@ module RelativePositioning if pos_before && (pos_after - pos_before) < 2 after.move_sequence_before + pos_before = after.prev_relative_position end self.relative_position = self.class.position_between(pos_before, pos_after) diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index bdd87437e2a..129d0fbb2c0 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -51,14 +51,21 @@ module Routable # Klass.where_full_path_in(%w{gitlab-org/gitlab-foss gitlab-org/gitlab}) # # Returns an ActiveRecord::Relation. - def where_full_path_in(paths) + def where_full_path_in(paths, use_includes: true) return none if paths.empty? wheres = paths.map do |path| "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" end - includes(:route).where(wheres.join(' OR ')).references(:routes) + route = + if use_includes + includes(:route).references(:routes) + else + joins(:route) + end + + route.where(wheres.join(' OR ')) end end diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index 3ff4b4046d3..10bbeecc2f7 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -80,4 +80,9 @@ module Spammable def check_for_spam? true end + + # Override in Spammable if differs + def allow_possible_spam? + Feature.enabled?(:allow_possible_spam, project) + end end diff --git a/app/models/concerns/stepable.rb b/app/models/concerns/stepable.rb index d00a049a004..dea241c5dbe 100644 --- a/app/models/concerns/stepable.rb +++ b/app/models/concerns/stepable.rb @@ -11,15 +11,15 @@ module Stepable initial_result = {} steps.inject(initial_result) do |previous_result, callback| - result = method(callback).call + result = method(callback).call(previous_result) - if result[:status] == :error - result[:failed_step] = callback + if result[:status] != :success + result[:last_step] = callback break result end - previous_result.merge(result) + result end end diff --git a/app/models/concerns/versioned_description.rb b/app/models/concerns/versioned_description.rb new file mode 100644 index 00000000000..63a24aadc8a --- /dev/null +++ b/app/models/concerns/versioned_description.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module VersionedDescription + extend ActiveSupport::Concern + + included do + attr_accessor :saved_description_version + + has_many :description_versions + + after_update :save_description_version + end + + private + + def save_description_version + self.saved_description_version = nil + + return unless Feature.enabled?(:save_description_versions, issuing_parent) + return unless saved_change_to_description? + + unless description_versions.exists? + description_versions.create!( + description: description_before_last_save, + created_at: created_at + ) + end + + self.saved_description_version = description_versions.create!(description: description) + end +end diff --git a/app/models/concerns/worker_attributes.rb b/app/models/concerns/worker_attributes.rb new file mode 100644 index 00000000000..af40e9e3b19 --- /dev/null +++ b/app/models/concerns/worker_attributes.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module WorkerAttributes + extend ActiveSupport::Concern + + class_methods do + def feature_category(value) + raise "Invalid category. Use `feature_category_not_owned!` to mark a worker as not owned" if value == :not_owned + + worker_attributes[:feature_category] = value + end + + # Special case: mark this work as not associated with a feature category + # this should be used for cross-cutting concerns, such as mailer workers. + def feature_category_not_owned! + worker_attributes[:feature_category] = :not_owned + end + + def get_feature_category + get_worker_attribute(:feature_category) + end + + def feature_category_not_owned? + get_worker_attribute(:feature_category) == :not_owned + end + + protected + + # Returns a worker attribute declared on this class or its parent class. + # This approach allows declared attributes to be inherited by + # child classes. + def get_worker_attribute(name) + worker_attributes[name] || superclass_worker_attributes(name) + end + + private + + def worker_attributes + @attributes ||= {} + end + + def superclass_worker_attributes(name) + return unless superclass.include? WorkerAttributes + + superclass.get_worker_attribute(name) + end + end +end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 583e23d1274..27bb76835c7 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -11,6 +11,7 @@ class ContainerRepository < ApplicationRecord delegate :client, to: :registry scope :ordered, -> { order(:name) } + scope :with_api_entity_associations, -> { preload(:project) } # rubocop: disable CodeReuse/ServiceClass def registry @@ -67,11 +68,9 @@ class ContainerRepository < ApplicationRecord def delete_tags! return unless has_tags? - digests = tags.map { |tag| tag.digest }.to_set + digests = tags.map { |tag| tag.digest }.compact.to_set - digests.all? do |digest| - delete_tag_by_digest(digest) - end + digests.map(&method(:delete_tag_by_digest)).all? end def delete_tag_by_digest(digest) diff --git a/app/models/cycle_analytics/project_level.rb b/app/models/cycle_analytics/project_level.rb index 4aa426c58a1..591435baf34 100644 --- a/app/models/cycle_analytics/project_level.rb +++ b/app/models/cycle_analytics/project_level.rb @@ -13,6 +13,7 @@ module CycleAnalytics def summary @summary ||= ::Gitlab::CycleAnalytics::StageSummary.new(project, from: options[:from], + to: options[:to], current_user: options[:current_user]).data end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index db7f9e06362..7ccd5e98360 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -9,7 +9,7 @@ class Deployment < ApplicationRecord belongs_to :environment, required: true belongs_to :cluster, class_name: 'Clusters::Cluster', optional: true belongs_to :user - belongs_to :deployable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + belongs_to :deployable, polymorphic: true, optional: true # rubocop:disable Cop/PolymorphicAssociations has_internal_id :iid, scope: :project, init: ->(s) do Deployment.where(project: s.project).maximum(:iid) if s&.project @@ -22,6 +22,8 @@ class Deployment < ApplicationRecord scope :for_environment, -> (environment) { where(environment_id: environment) } + scope :visible, -> { where(status: %i[running success failed canceled]) } + state_machine :status, initial: :created do event :run do transition created: :running @@ -73,6 +75,10 @@ class Deployment < ApplicationRecord find(ids) end + def self.find_successful_deployment!(iid) + success.find_by!(iid: iid) + end + def commit project.commit(sha) end @@ -180,3 +186,5 @@ class Deployment < ApplicationRecord self.created_at if success? && !read_attribute(:finished_at) end end + +Deployment.prepend_if_ee('EE::Deployment') diff --git a/app/models/description_version.rb b/app/models/description_version.rb new file mode 100644 index 00000000000..abab7f94212 --- /dev/null +++ b/app/models/description_version.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class DescriptionVersion < ApplicationRecord + belongs_to :issue + belongs_to :merge_request + + validate :exactly_one_issuable + + def self.issuable_attrs + %i(issue merge_request).freeze + end + + private + + def exactly_one_issuable + issuable_count = self.class.issuable_attrs.count { |attr| self["#{attr}_id"] } + + errors.add(:base, "Exactly one of #{self.class.issuable_attrs.join(', ')} is required") if issuable_count != 1 + end +end + +DescriptionVersion.prepend_if_ee('EE::DescriptionVersion') diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index aa7286a9971..65e87bb08a7 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -75,6 +75,10 @@ class DiffNote < Note self.original_position.diff_refs == diff_refs end + # Checks if the current `position` line in the diff + # exists and is suggestible (not a deletion). + # + # Avoid using in iterations as it requests Gitaly. def supports_suggestion? return false unless noteable&.supports_suggestion? && on_text? # We don't want to trigger side-effects of `diff_file` call. diff --git a/app/models/diff_viewer/image.rb b/app/models/diff_viewer/image.rb index 350bef1d42a..cfda0058d81 100644 --- a/app/models/diff_viewer/image.rb +++ b/app/models/diff_viewer/image.rb @@ -6,7 +6,7 @@ module DiffViewer include ClientSide self.partial_name = 'image' - self.extensions = UploaderHelper::IMAGE_EXT + self.extensions = UploaderHelper::SAFE_IMAGE_EXT self.binary = true self.switcher_icon = 'picture-o' self.switcher_title = _('image diff') diff --git a/app/models/environment.rb b/app/models/environment.rb index fe438b142b2..af0c219d9a0 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -6,7 +6,8 @@ class Environment < ApplicationRecord belongs_to :project, required: true - has_many :deployments, -> { success }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :deployments, -> { visible }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :successful_deployments, -> { success }, class_name: 'Deployment' has_one :last_deployment, -> { success.order('deployments.id DESC') }, class_name: 'Deployment' @@ -81,6 +82,10 @@ class Environment < ApplicationRecord pluck(:name) end + def self.find_or_create_by_name(name) + find_or_create_by(name: name) + end + def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_ENVIRONMENT_NAME', value: name) diff --git a/app/models/event.rb b/app/models/event.rb index 205e1f71c74..9611019adb8 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -77,15 +77,6 @@ class Event < ApplicationRecord scope :recent, -> { reorder(id: :desc) } scope :code_push, -> { where(action: PUSHED) } - scope :in_projects, -> (projects) do - sub_query = projects - .except(:order) - .select(1) - .where('projects.id = events.project_id') - - where('EXISTS (?)', sub_query).recent - end - scope :with_associations, -> do # We're using preload for "push_event_payload" as otherwise the association # is not always available (depending on the query being built). diff --git a/app/models/event_collection.rb b/app/models/event_collection.rb index a4c69b11781..4778f74568e 100644 --- a/app/models/event_collection.rb +++ b/app/models/event_collection.rb @@ -6,6 +6,8 @@ # in a controller), it's not suitable for building queries that are used for # building other queries. class EventCollection + include Gitlab::Utils::StrongMemoize + # To prevent users from putting too much pressure on the database by cycling # through thousands of events we put a limit on the number of pages. MAX_PAGE = 10 @@ -13,57 +15,52 @@ class EventCollection # projects - An ActiveRecord::Relation object that returns the projects for # which to retrieve events. # filter - An EventFilter instance to use for filtering events. - def initialize(projects, limit: 20, offset: 0, filter: nil) + def initialize(projects, limit: 20, offset: 0, filter: nil, groups: nil) @projects = projects @limit = limit @offset = offset @filter = filter + @groups = groups end # Returns an Array containing the events. def to_a return [] if current_page > MAX_PAGE - relation = if Gitlab::Database.join_lateral_supported? - relation_with_join_lateral + relation = if groups + project_and_group_events else - relation_without_join_lateral + relation_with_join_lateral('project_id', projects) end + relation = paginate_events(relation) relation.with_associations.to_a end private - # Returns the events relation to use when JOIN LATERAL is not supported. - # - # This relation simply gets all the events for all authorized projects, then - # limits that set. - def relation_without_join_lateral - events = filtered_events.in_projects(projects) + def project_and_group_events + project_events = relation_with_join_lateral('project_id', projects) + group_events = relation_with_join_lateral('group_id', groups) - paginate_events(events) + Event.from_union([project_events, group_events]).recent end - # Returns the events relation to use when JOIN LATERAL is supported. - # # This relation is built using JOIN LATERAL, producing faster queries than a # regular LIMIT + OFFSET approach. - def relation_with_join_lateral - projects_for_lateral = projects.select(:id).to_sql + def relation_with_join_lateral(parent_column, parents) + parents_for_lateral = parents.select(:id).to_sql lateral = filtered_events .limit(limit_for_join_lateral) - .where('events.project_id = projects_for_lateral.id') + .where("events.#{parent_column} = parents_for_lateral.id") # rubocop:disable GitlabSecurity/SqlInjection .to_sql # The outer query does not need to re-apply the filters since the JOIN # LATERAL body already takes care of this. - outer = base_relation - .from("(#{projects_for_lateral}) projects_for_lateral") + base_relation + .from("(#{parents_for_lateral}) parents_for_lateral") .joins("JOIN LATERAL (#{lateral}) AS #{Event.table_name} ON true") - - paginate_events(outer) end def filtered_events @@ -97,4 +94,10 @@ class EventCollection def projects @projects.except(:order) end + + def groups + strong_memoize(:groups) do + groups.except(:order) if @groups + end + end end diff --git a/app/models/evidence.rb b/app/models/evidence.rb new file mode 100644 index 00000000000..69a00f1cb3f --- /dev/null +++ b/app/models/evidence.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class Evidence < ApplicationRecord + include ShaAttribute + + belongs_to :release + + before_validation :generate_summary_and_sha + + default_scope { order(created_at: :asc) } + + sha_attribute :summary_sha + + def milestones + @milestones ||= release.milestones.includes(:issues) + end + + private + + def generate_summary_and_sha + summary = Evidences::EvidenceSerializer.new.represent(self) # rubocop: disable CodeReuse/Serializer + return unless summary + + self.summary = summary + self.summary_sha = Gitlab::CryptoHelper.sha256(summary) + end +end diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb index 1d553fc8312..7d766e1f25c 100644 --- a/app/models/global_milestone.rb +++ b/app/models/global_milestone.rb @@ -11,7 +11,7 @@ class GlobalMilestone delegate :title, :state, :due_date, :start_date, :participants, :project, :group, :expires_at, :closed?, :iid, :group_milestone?, :safe_title, - :milestoneish_id, :parent, to: :milestone + :milestoneish_id, :resource_parent, to: :milestone def to_hash { diff --git a/app/models/gpg_signature.rb b/app/models/gpg_signature.rb index 46cac1d41bb..0c36e51120f 100644 --- a/app/models/gpg_signature.rb +++ b/app/models/gpg_signature.rb @@ -23,6 +23,8 @@ class GpgSignature < ApplicationRecord validates :project_id, presence: true validates :gpg_key_primary_keyid, presence: true + scope :by_commit_sha, ->(shas) { where(commit_sha: shas) } + def self.with_key_and_subkeys(gpg_key) subkey_ids = gpg_key.subkeys.pluck(:id) diff --git a/app/models/grafana_integration.rb b/app/models/grafana_integration.rb new file mode 100644 index 00000000000..51cc398394d --- /dev/null +++ b/app/models/grafana_integration.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class GrafanaIntegration < ApplicationRecord + belongs_to :project + + attr_encrypted :token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32 + + validates :grafana_url, + length: { maximum: 1024 }, + addressable_url: { enforce_sanitization: true, ascii_only: true } + + validates :token, :project, presence: true + + def client + @client ||= ::Grafana::Client.new(api_url: grafana_url.chomp('/'), token: token) + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 1b62db04ab7..042201ffa14 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -14,6 +14,7 @@ class Group < Namespace include TokenAuthenticatable include WithUploads include Gitlab::Utils::StrongMemoize + include GroupAPICompatibility ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 @@ -258,6 +259,10 @@ class Group < Namespace members_with_parents.maintainers.exists?(user_id: user) end + def has_container_repositories? + container_repositories.exists? + end + # @deprecated alias_method :has_master?, :has_maintainer? @@ -435,6 +440,10 @@ class Group < Namespace members.owners.order_recent_sign_in.limit(ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end + def supports_events? + false + end + private def update_two_factor_requirement @@ -464,6 +473,12 @@ class Group < Namespace errors.add(:visibility_level, "#{visibility} is not allowed since there are sub-groups with higher visibility.") end + + def self.groups_including_descendants_by(group_ids) + Gitlab::ObjectHierarchy + .new(Group.where(id: group_ids)) + .base_and_descendants + end end Group.prepend_if_ee('EE::Group') diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 16fc7fdbd48..e51b1c41059 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -13,7 +13,7 @@ class WebHook < ApplicationRecord algorithm: 'aes-256-gcm', key: Settings.attr_encrypted_db_key_base_32 - has_many :web_hook_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :web_hook_logs validates :url, presence: true validates :url, public_url: true, unless: ->(hook) { hook.is_a?(SystemHook) } diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index 237401899db..8d3eeaf2461 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -16,6 +16,8 @@ # * Add `usage` value to enum # * (Optionally) add columns to `internal_ids` if needed for scope. class InternalId < ApplicationRecord + include Gitlab::Utils::StrongMemoize + belongs_to :project belongs_to :namespace @@ -47,10 +49,18 @@ class InternalId < ApplicationRecord def update_and_save(&block) lock! yield + update_and_save_counter.increment(usage: usage, changed: last_value_changed?) save! last_value end + # Instrumentation to track for-update locks + def update_and_save_counter + strong_memoize(:update_and_save_counter) do + Gitlab::Metrics.counter(:gitlab_internal_id_for_update_lock, 'Number of ROW SHARE (FOR UPDATE) locks on individual records from internal_ids') + end + end + class << self def track_greatest(subject, scope, usage, new_value, init) return new_value unless available? diff --git a/app/models/issue.rb b/app/models/issue.rb index d0b2165fcc7..b9b481ac29b 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -71,7 +71,7 @@ class Issue < ApplicationRecord attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true - state_machine :state, initial: :opened do + state_machine :state_id, initial: :opened do event :close do transition [:opened] => :closed end @@ -80,8 +80,8 @@ class Issue < ApplicationRecord transition closed: :opened end - state :opened - state :closed + state :opened, value: Issue.available_states[:opened] + state :closed, value: Issue.available_states[:closed] before_transition any => :closed do |issue| issue.closed_at = issue.system_note_timestamp @@ -93,6 +93,13 @@ class Issue < ApplicationRecord end end + # Alias to state machine .with_state_id method + # This needs to be defined after the state machine block to avoid errors + class << self + alias_method :with_state, :with_state_id + alias_method :with_states, :with_state_ids + end + def self.relative_positioning_query_base(issue) in_projects(issue.parent_ids) end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 60b11ad9356..535c3cf2ba1 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -2,6 +2,7 @@ class LfsObject < ApplicationRecord include AfterCommitQueue + include Checksummable include EachBatch include ObjectStorage::BackgroundMove @@ -9,6 +10,7 @@ class LfsObject < ApplicationRecord has_many :projects, -> { distinct }, through: :lfs_objects_projects scope :with_files_stored_locally, -> { where(file_store: LfsObjectUploader::Store::LOCAL) } + scope :with_files_stored_remotely, -> { where(file_store: LfsObjectUploader::Store::REMOTE) } validates :oid, presence: true, uniqueness: true @@ -23,7 +25,13 @@ class LfsObject < ApplicationRecord end def project_allowed_access?(project) - projects.exists?(project.lfs_storage_project.id) + if project.fork_network_member + lfs_objects_projects + .where("EXISTS(?)", project.fork_network.fork_network_members.select(1).where("fork_network_members.project_id = lfs_objects_projects.project_id")) + .exists? + else + lfs_objects_projects.where(project_id: project.id).exists? + end end def local_store? @@ -39,7 +47,7 @@ class LfsObject < ApplicationRecord # rubocop: enable DestroyAll def self.calculate_oid(path) - Digest::SHA256.file(path).hexdigest + self.hexdigest(path) end end diff --git a/app/models/list.rb b/app/models/list.rb index b4a4631b397..13c42b55bf7 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -21,20 +21,10 @@ class List < ApplicationRecord scope :destroyable, -> { where(list_type: list_types.slice(*destroyable_types).values) } scope :movable, -> { where(list_type: list_types.slice(*movable_types).values) } - scope :preload_associations, -> (user) do - preload(:board, label: :priorities) - end + scope :preload_associations, -> { preload(:board, label: :priorities) } scope :ordered, -> { order(:list_type, :position) } - # Loads list with preferences for given user - # if preferences exists for user or not - scope :with_preferences_for, -> (user) do - return unless user - - includes(:list_user_preferences).where(list_user_preferences: { user_id: [user.id, nil] }) - end - alias_method :preferences, :list_user_preferences class << self @@ -45,25 +35,25 @@ class List < ApplicationRecord def movable_types [:label] end + + def preload_preferences_for_user(lists, user) + return unless user + + lists.each { |list| list.preferences_for(user) } + end end def preferences_for(user) return preferences.build unless user - if preferences.loaded? - preloaded_preferences_for(user) - else - preferences.find_or_initialize_by(user: user) - end - end + BatchLoader.for(list_id: id, user_id: user.id).batch(default_value: preferences.build(user: user)) do |items, loader| + list_ids = items.map { |i| i[:list_id] } + user_ids = items.map { |i| i[:user_id] } - def preloaded_preferences_for(user) - user_preferences = - preferences.find do |preference| - preference.user_id == user.id + ListUserPreference.where(list_id: list_ids.uniq, user_id: user_ids.uniq).find_each do |preference| + loader.call({ list_id: preference.list_id, user_id: preference.user_id }, preference) end - - user_preferences || preferences.build(user: user) + end end def update_preferences_for(user, preferences = {}) diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 63133ca285b..7cdaa3e3ca7 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -85,7 +85,13 @@ class MergeRequest < ApplicationRecord # when creating new merge request attr_accessor :can_be_created, :compare_commits, :diff_options, :compare - state_machine :state, initial: :opened do + # Keep states definition to be evaluated before the state_machine block to avoid spec failures. + # If this gets evaluated after, the `merged` and `locked` states which are overrided can be nil. + def self.available_state_names + super + [:merged, :locked] + end + + state_machine :state_id, initial: :opened do event :close do transition [:opened] => :closed end @@ -116,10 +122,17 @@ class MergeRequest < ApplicationRecord end end - state :opened - state :closed - state :merged - state :locked + state :opened, value: MergeRequest.available_states[:opened] + state :closed, value: MergeRequest.available_states[:closed] + state :merged, value: MergeRequest.available_states[:merged] + state :locked, value: MergeRequest.available_states[:locked] + end + + # Alias to state machine .with_state_id method + # This needs to be defined after the state machine block to avoid errors + class << self + alias_method :with_state, :with_state_id + alias_method :with_states, :with_state_ids end state_machine :merge_status, initial: :unchecked do @@ -196,6 +209,10 @@ class MergeRequest < ApplicationRecord scope :by_target_branch, ->(branch_name) { where(target_branch: branch_name) } scope :preload_source_project, -> { preload(:source_project) } + scope :with_open_merge_when_pipeline_succeeds, -> do + with_state(:opened).where(merge_when_pipeline_succeeds: true) + end + after_save :keep_around_commit alias_attribute :project, :target_project @@ -207,10 +224,6 @@ class MergeRequest < ApplicationRecord '!' end - def self.available_states - @available_states ||= super.merge(merged: 3, locked: 4) - end - # Returns the top 100 target branches # # The returned value is a Array containing branch names @@ -450,6 +463,15 @@ class MergeRequest < ApplicationRecord merge_request_diffs.where.not(id: merge_request_diff.id) end + # Overwritten in EE + def note_positions_for_paths(paths, _user = nil) + positions = notes.new_diff_notes.joins(:note_diff_file) + .where('note_diff_files.old_path IN (?) OR note_diff_files.new_path IN (?)', paths, paths) + .positions + + Gitlab::Diff::PositionCollection.new(positions, diff_head_sha) + end + def preloads_discussion_diff_highlighting? true end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 8b5f10ce159..735ad046f22 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -83,7 +83,7 @@ class MergeRequestDiff < ApplicationRecord metrics_join = mr_diffs.join(mr_metrics).on(metrics_join_condition) - condition = MergeRequest.arel_table[:state].eq(:merged) + condition = MergeRequest.arel_table[:state_id].eq(MergeRequest.available_states[:merged]) .and(MergeRequest::Metrics.arel_table[:merged_at].lteq(before)) .and(MergeRequest::Metrics.arel_table[:merged_at].not_eq(nil)) @@ -91,7 +91,7 @@ class MergeRequestDiff < ApplicationRecord end scope :old_closed_diffs, -> (before) do - condition = MergeRequest.arel_table[:state].eq(:closed) + condition = MergeRequest.arel_table[:state_id].eq(MergeRequest.available_states[:closed]) .and(MergeRequest::Metrics.arel_table[:latest_closed_at].lteq(before)) joins(merge_request: :metrics).where(condition) @@ -136,6 +136,7 @@ class MergeRequestDiff < ApplicationRecord # All diff information is collected from repository after object is created. # It allows you to override variables like head_commit_sha before getting diff. after_create :save_git_content, unless: :importing? + after_create_commit :set_as_latest_diff after_save :update_external_diff_store, if: -> { !importing? && saved_change_to_external_diff? } @@ -150,10 +151,6 @@ class MergeRequestDiff < ApplicationRecord # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content - MergeRequest - .where('id = ? AND COALESCE(latest_merge_request_diff_id, 0) < ?', self.merge_request_id, self.id) - .update_all(latest_merge_request_diff_id: self.id) - ensure_commit_shas save_commits save_diffs @@ -168,6 +165,12 @@ class MergeRequestDiff < ApplicationRecord keep_around_commits end + def set_as_latest_diff + MergeRequest + .where('id = ? AND COALESCE(latest_merge_request_diff_id, 0) < ?', self.merge_request_id, self.id) + .update_all(latest_merge_request_diff_id: self.id) + end + def ensure_commit_shas self.start_commit_sha ||= merge_request.target_branch_sha self.head_commit_sha ||= merge_request.source_branch_sha @@ -297,6 +300,13 @@ class MergeRequestDiff < ApplicationRecord base_commit_sha? && head_commit_sha? && start_commit_sha? end + def diffs_in_batch(batch_page, batch_size, diff_options:) + Gitlab::Diff::FileCollection::MergeRequestDiffBatch.new(self, + batch_page, + batch_size, + diff_options: diff_options) + end + def diffs(diff_options = nil) if without_files? && comparison = diff_refs&.compare_in(project) # It should fetch the repository when diffs are cleaned by the system. @@ -495,11 +505,6 @@ class MergeRequestDiff < ApplicationRecord merge_request.closed? && merge_request.metrics.latest_closed_at < EXTERNAL_DIFF_CUTOFF.ago end - # We can't rely on `merge_request.latest_merge_request_diff_id` because that - # may have been changed in `save_git_content` without being reflected in - # the association's instance. This query is always subject to races, but - # the worst case is that we *don't* make a diff external when we could. The - # background worker will make it external at a later date. def old_version? latest_id = MergeRequest .where(id: merge_request_id) @@ -507,7 +512,7 @@ class MergeRequestDiff < ApplicationRecord .pluck(:latest_merge_request_diff_id) .first - self.id != latest_id + latest_id && self.id < latest_id end def load_diffs(options) @@ -584,3 +589,5 @@ class MergeRequestDiff < ApplicationRecord end end end + +MergeRequestDiff.prepend_if_ee('EE::MergeRequestDiff') diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index a532c1e6356..14c86ec69da 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -5,6 +5,7 @@ class MergeRequestDiffFile < ApplicationRecord include DiffFile belongs_to :merge_request_diff, inverse_of: :merge_request_diff_files + alias_attribute :index, :relative_order def utf8_diff return '' if diff.blank? diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 916c11a8d03..2fa0cfc9b93 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -257,10 +257,9 @@ class Milestone < ApplicationRecord title.to_slug.normalize.to_s end - def parent + def resource_parent group || project end - alias_method :resource_parent, :parent def group_milestone? group_id.present? diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9a7c3dc03c3..5663ebf8ba1 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -120,6 +120,13 @@ class Namespace < ApplicationRecord uniquify = Uniquify.new uniquify.string(path) { |s| Namespace.find_by_path_or_name(s) } end + + def find_by_pages_host(host) + gitlab_host = "." + Settings.pages.host.downcase + name = host.downcase.delete_suffix(gitlab_host) + + Namespace.find_by_full_path(name) + end end def visibility_level_field @@ -175,7 +182,7 @@ class Namespace < ApplicationRecord # any ancestor can disable emails for all descendants def emails_disabled? strong_memoize(:emails_disabled) do - Feature.enabled?(:emails_disabled, self, default_enabled: true) && self_and_ancestors.where(emails_disabled: true).exists? + self_and_ancestors.where(emails_disabled: true).exists? end end @@ -249,7 +256,7 @@ class Namespace < ApplicationRecord end def has_parent? - parent.present? + parent_id.present? || parent.present? end def root_ancestor @@ -305,8 +312,28 @@ class Namespace < ApplicationRecord aggregation_schedule.present? end + def pages_virtual_domain + Pages::VirtualDomain.new(all_projects_with_pages, trim_prefix: full_path) + end + + def closest_setting(name) + self_and_ancestors(hierarchy_order: :asc) + .find { |n| !n.read_attribute(name).nil? } + .try(name) + end + private + def all_projects_with_pages + if all_projects.pages_metadata_not_migrated.exists? + Gitlab::BackgroundMigration::MigratePagesMetadata.new.perform_on_relation( + all_projects.pages_metadata_not_migrated + ) + end + + all_projects.with_pages_deployed + end + def parent_changed? parent_id_changed? end diff --git a/app/models/note.rb b/app/models/note.rb index b1829e71017..43f349c6fa2 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -24,7 +24,7 @@ class Note < ApplicationRecord class << self def values - constants.map {|const| self.const_get(const)} + constants.map {|const| self.const_get(const, false)} end def value?(val) @@ -104,6 +104,8 @@ class Note < ApplicationRecord end end + validate :does_not_exceed_notes_limit?, on: :create, unless: [:system?, :importing?] + # @deprecated attachments are handler by the MarkdownUploader mount_uploader :attachment, AttachmentUploader @@ -143,6 +145,9 @@ class Note < ApplicationRecord end scope :with_metadata, -> { includes(:system_note_metadata) } + scope :for_note_or_capitalized_note, ->(text) { where(note: [text, text.capitalize]) } + scope :like_note_or_capitalized_note, ->(text) { where('(note LIKE ? OR note LIKE ?)', text, text.capitalize) } + after_initialize :ensure_discussion_id before_validation :nullify_blank_type, :nullify_blank_line_code before_validation :set_discussion_id, on: :create @@ -193,6 +198,12 @@ class Note < ApplicationRecord groups end + def positions + where.not(position: nil) + .select(:id, :type, :position) # ActiveRecord needs id and type for typecasting. + .map(&:position) + end + def count_for_collection(ids, type) user.select('noteable_id', 'COUNT(*) as count') .group(:noteable_id) @@ -215,7 +226,7 @@ class Note < ApplicationRecord if force_cross_reference_regex_check? matches_cross_reference_regex? else - SystemNoteService.cross_reference?(note) + ::SystemNotes::IssuablesService.cross_reference?(note) end end # rubocop: enable CodeReuse/ServiceClass @@ -472,10 +483,9 @@ class Note < ApplicationRecord Upload.find_by(model: self, path: paths) end - def parent + def resource_parent project end - alias_method :resource_parent, :parent private @@ -519,6 +529,12 @@ class Note < ApplicationRecord system_note_metadata&.cross_reference_types&.include?(system_note_metadata&.action) end + + def does_not_exceed_notes_limit? + return unless noteable + + errors.add(:base, _('Maximum number of comments exceeded')) if noteable.notes.count >= Noteable::MAX_NOTES_LIMIT + end end Note.prepend_if_ee('EE::Note') diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 981590b688f..2b3443f24d7 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -25,6 +25,7 @@ class NotificationSetting < ApplicationRecord end EMAIL_EVENTS = [ + :new_release, :new_note, :new_issue, :reopen_issue, @@ -46,6 +47,10 @@ class NotificationSetting < ApplicationRecord EMAIL_EVENTS end + def self.allowed_fields(source = nil) + NotificationSetting.email_events(source).dup + %i(level notification_email) + end + def email_events self.class.email_events(source) end diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index 1b3183a2a43..51c496c77d3 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -2,9 +2,10 @@ module Pages class LookupPath - def initialize(project, domain: nil) + def initialize(project, trim_prefix: nil, domain: nil) @project = project @domain = domain + @trim_prefix = trim_prefix || project.full_path end def project_id @@ -28,11 +29,15 @@ module Pages end def prefix - '/' + if project.pages_group_root? + '/' + else + project.full_path.delete_prefix(trim_prefix) + '/' + end end private - attr_reader :project, :domain + attr_reader :project, :trim_prefix, :domain end end diff --git a/app/models/pages/virtual_domain.rb b/app/models/pages/virtual_domain.rb index 3a876dc06a2..7e42b8e6ae2 100644 --- a/app/models/pages/virtual_domain.rb +++ b/app/models/pages/virtual_domain.rb @@ -2,8 +2,9 @@ module Pages class VirtualDomain - def initialize(projects, domain: nil) + def initialize(projects, trim_prefix: nil, domain: nil) @projects = projects + @trim_prefix = trim_prefix @domain = domain end @@ -17,12 +18,12 @@ module Pages def lookup_paths projects.map do |project| - project.pages_lookup_path(domain: domain) + project.pages_lookup_path(trim_prefix: trim_prefix, domain: domain) end.sort_by(&:prefix).reverse end private - attr_reader :projects, :domain + attr_reader :projects, :trim_prefix, :domain end end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 22a6bae7cf7..7903a2182dd 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -186,11 +186,27 @@ class PagesDomain < ApplicationRecord end def pages_virtual_domain + return unless pages_deployed? + Pages::VirtualDomain.new([project], domain: self) end private + def pages_deployed? + # TODO: remove once `pages_metadatum` is migrated + # https://gitlab.com/gitlab-org/gitlab/issues/33106 + unless project.pages_metadatum + Gitlab::BackgroundMigration::MigratePagesMetadata + .new + .perform_on_relation(Project.where(id: project_id)) + + project.reset + end + + project.pages_metadatum&.deployed? + end + def set_verification_code return if self.verification_code.present? diff --git a/app/models/project.rb b/app/models/project.rb index 5c3bf4a3b5d..3525f37f8d5 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -68,7 +68,7 @@ class Project < ApplicationRecord :snippets_access_level, :builds_access_level, :repository_access_level, to: :project_feature, allow_nil: true - delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage + delegate :base_dir, :disk_path, to: :storage delegate :scheduled?, :started?, :in_progress?, :failed?, :finished?, @@ -104,6 +104,9 @@ class Project < ApplicationRecord unless: :ci_cd_settings, if: proc { ProjectCiCdSetting.available? } + after_create :create_pages_metadatum, + unless: :pages_metadatum + after_create :set_timestamps_for_create after_update :update_forks_visibility_level @@ -119,8 +122,6 @@ class Project < ApplicationRecord # Storage specific hooks after_initialize :use_hashed_storage after_create :check_repository_absence! - after_create :ensure_storage_path_exists - after_save :ensure_storage_path_exists, if: :saved_change_to_namespace_id? acts_as_ordered_taggable @@ -192,6 +193,7 @@ class Project < ApplicationRecord has_one :project_repository, inverse_of: :project has_one :error_tracking_setting, inverse_of: :project, class_name: 'ErrorTracking::ProjectErrorTrackingSetting' has_one :metrics_setting, inverse_of: :project, class_name: 'ProjectMetricsSetting' + has_one :grafana_integration, inverse_of: :project # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id', inverse_of: :target_project @@ -242,8 +244,8 @@ class Project < ApplicationRecord has_one :cluster_project, class_name: 'Clusters::Project' has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' - has_many :cluster_ingresses, through: :clusters, source: :application_ingress, class_name: 'Clusters::Applications::Ingress' has_many :kubernetes_namespaces, class_name: 'Clusters::KubernetesNamespace' + has_many :management_clusters, class_name: 'Clusters::Cluster', foreign_key: :management_project_id, inverse_of: :management_project has_many :prometheus_metrics @@ -273,12 +275,13 @@ class Project < ApplicationRecord has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks + has_many :job_artifacts, class_name: 'Ci::JobArtifact' has_many :runner_projects, class_name: 'Ci::RunnerProject', inverse_of: :project has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' has_many :environments - has_many :deployments, -> { success } + has_many :deployments has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' has_many :project_deploy_tokens has_many :deploy_tokens, through: :project_deploy_tokens @@ -294,6 +297,11 @@ class Project < ApplicationRecord has_many :external_pull_requests, inverse_of: :project + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_project_id + has_many :source_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :project_id + + has_one :pages_metadatum, class_name: 'ProjectPagesMetadatum', inverse_of: :project + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data @@ -306,6 +314,7 @@ class Project < ApplicationRecord accepts_nested_attributes_for :error_tracking_setting, update_only: true accepts_nested_attributes_for :metrics_setting, update_only: true, allow_destroy: true + accepts_nested_attributes_for :grafana_integration, update_only: true, allow_destroy: true delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true @@ -424,6 +433,15 @@ class Project < ApplicationRecord .where(project_ci_cd_settings: { group_runners_enabled: true }) end + scope :with_pages_deployed, -> do + joins(:pages_metadatum).merge(ProjectPagesMetadatum.deployed) + end + + scope :pages_metadata_not_migrated, -> do + left_outer_joins(:pages_metadatum) + .where(project_pages_metadata: { project_id: nil }) + end + enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, @@ -650,7 +668,7 @@ class Project < ApplicationRecord def emails_disabled? strong_memoize(:emails_disabled) do # disabling in the namespace overrides the project setting - Feature.enabled?(:emails_disabled, self, default_enabled: true) && (super || namespace.emails_disabled?) + super || namespace.emails_disabled? end end @@ -1018,8 +1036,8 @@ class Project < ApplicationRecord end end - def web_url - Gitlab::Routing.url_helpers.project_url(self) + def web_url(only_path: nil) + Gitlab::Routing.url_helpers.project_url(self, only_path: only_path) end def readme_url @@ -1298,7 +1316,18 @@ class Project < ApplicationRecord end def http_url_to_repo - "#{web_url}.git" + custom_root = Gitlab::CurrentSettings.custom_http_clone_url_root + + project_url = if custom_root.present? + Gitlab::Utils.append_path( + custom_root, + web_url(only_path: true) + ) + else + web_url + end + + "#{project_url}.git" end # Is overridden in EE @@ -1647,6 +1676,10 @@ class Project < ApplicationRecord "#{url}/#{url_path}" end + def pages_group_root? + pages_group_url == pages_url + end + def pages_subdomain full_path.partition('/').first end @@ -1685,6 +1718,7 @@ class Project < ApplicationRecord # Projects with a missing namespace cannot have their pages removed return unless namespace + mark_pages_as_not_deployed unless destroyed? ::Projects::UpdatePagesConfigurationService.new(self).execute # 1. We rename pages to temporary directory @@ -1698,6 +1732,14 @@ class Project < ApplicationRecord end # rubocop: enable CodeReuse/ServiceClass + def mark_pages_as_deployed + ensure_pages_metadatum.update!(deployed: true) + end + + def mark_pages_as_not_deployed + ensure_pages_metadatum.update!(deployed: false) + end + # rubocop:disable Gitlab/RailsLogger def write_repository_config(gl_full_path: full_path) # We'd need to keep track of project full path otherwise directory tree @@ -1821,6 +1863,7 @@ class Project < ApplicationRecord Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PROJECT_ID', value: id.to_s) .append(key: 'CI_PROJECT_NAME', value: path) + .append(key: 'CI_PROJECT_TITLE', value: title) .append(key: 'CI_PROJECT_PATH', value: full_path) .append(key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug) .append(key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path) @@ -2217,12 +2260,37 @@ class Project < ApplicationRecord members.maintainers.order_recent_sign_in.limit(ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end - def pages_lookup_path(domain: nil) - Pages::LookupPath.new(self, domain: domain) + def pages_lookup_path(trim_prefix: nil, domain: nil) + Pages::LookupPath.new(self, trim_prefix: trim_prefix, domain: domain) + end + + def closest_setting(name) + setting = read_attribute(name) + setting = closest_namespace_setting(name) if setting.nil? + setting = app_settings_for(name) if setting.nil? + setting + end + + def drop_visibility_level! + if group && group.visibility_level < visibility_level + self.visibility_level = group.visibility_level + end + + if Gitlab::CurrentSettings.restricted_visibility_levels.include?(visibility_level) + self.visibility_level = Gitlab::VisibilityLevel::PRIVATE + end end private + def closest_namespace_setting(name) + namespace.closest_setting(name) + end + + def app_settings_for(name) + Gitlab::CurrentSettings.send(name) # rubocop:disable GitlabSecurity/PublicSend + end + def merge_requests_allowing_collaboration(source_branch = nil) relation = source_of_merge_requests.opened.where(allow_collaboration: true) relation = relation.where(source_branch: source_branch) if source_branch @@ -2268,7 +2336,7 @@ class Project < ApplicationRecord end def repository_with_same_path_already_exists? - gitlab_shell.exists?(repository_storage, "#{disk_path}.git") + gitlab_shell.repository_exists?(repository_storage, "#{disk_path}.git") end def set_timestamps_for_create @@ -2346,6 +2414,13 @@ class Project < ApplicationRecord def services_templates @services_templates ||= Service.where(template: true) end + + def ensure_pages_metadatum + pages_metadatum || create_pages_metadatum! + rescue ActiveRecord::RecordNotUnique + reset + retry + end end Project.prepend_if_ee('EE::Project') diff --git a/app/models/project_pages_metadatum.rb b/app/models/project_pages_metadatum.rb new file mode 100644 index 00000000000..1fda388b1ae --- /dev/null +++ b/app/models/project_pages_metadatum.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class ProjectPagesMetadatum < ApplicationRecord + self.primary_key = :project_id + + belongs_to :project, inverse_of: :pages_metadatum + + scope :deployed, -> { where(deployed: true) } +end diff --git a/app/models/project_services/data_fields.rb b/app/models/project_services/data_fields.rb index 46136556ade..cffb493d569 100644 --- a/app/models/project_services/data_fields.rb +++ b/app/models/project_services/data_fields.rb @@ -5,7 +5,7 @@ module DataFields class_methods do # Provide convenient accessor methods for data fields. - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab-foss/issues/63084 + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 def data_field(*args) args.each do |arg| self.class_eval <<-RUBY, __FILE__, __LINE__ + 1 diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index 3320405e9e9..019bd54f48c 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -73,7 +73,7 @@ class HipchatService < Service private def gate - options = { api_version: api_version.present? ? api_version : 'v2' } + options = { api_version: api_version.presence || 'v2' } options[:server_url] = server unless server.blank? @gate ||= HipChat::Client.new(token, options) end @@ -161,7 +161,7 @@ class HipchatService < Service obj_attr = data[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) title = render_line(obj_attr[:title]) - state = obj_attr[:state] + state = Issue.available_states.key(obj_attr[:state_id]) issue_iid = obj_attr[:iid] issue_url = obj_attr[:url] description = obj_attr[:description] diff --git a/app/models/project_services/irker_service.rb b/app/models/project_services/irker_service.rb index fb76bc89c98..4a6c8339625 100644 --- a/app/models/project_services/irker_service.rb +++ b/app/models/project_services/irker_service.rb @@ -36,8 +36,8 @@ class IrkerService < Service def settings { - server_host: server_host.present? ? server_host : 'localhost', - server_port: server_port.present? ? server_port : 6659 + server_host: server_host.presence || 'localhost', + server_port: server_port.presence || 6659 } end diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index 3ecd5390d79..9e1393196ff 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -4,7 +4,7 @@ class IssueTrackerService < Service validate :one_issue_tracker, if: :activated?, on: :manual_change # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab-foss/issues/63084 + # https://gitlab.com/gitlab-org/gitlab/issues/29404 data_field :project_url, :issues_url, :new_issue_url default_value_for :category, 'issue_tracker' @@ -25,7 +25,7 @@ class IssueTrackerService < Service end end - # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-foss/issues/63084 + # this will be removed as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 def title if title_attribute = read_attribute(:title) title_attribute @@ -36,7 +36,7 @@ class IssueTrackerService < Service end end - # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-foss/issues/63084 + # this will be removed as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 def description if description_attribute = read_attribute(:description) description_attribute @@ -49,7 +49,7 @@ class IssueTrackerService < Service def handle_properties # this has been moved from initialize_properties and should be improved - # as part of https://gitlab.com/gitlab-org/gitlab-foss/issues/63084 + # as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 return unless properties @legacy_properties_data = properties.dup @@ -62,6 +62,7 @@ class IssueTrackerService < Service end data_values.reject! { |key| data_fields.changed.include?(key) } + data_values.slice!(*data_fields.attributes.keys) data_fields.assign_attributes(data_values) if data_values.present? self.properties = {} @@ -71,6 +72,10 @@ class IssueTrackerService < Service @legacy_properties_data ||= {} end + def supports_data_fields? + true + end + def data_fields issue_tracker_data || self.build_issue_tracker_data end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index a76970bfa2a..ba61810e26f 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -19,7 +19,7 @@ class JiraService < IssueTrackerService # for more information check: https://gitlab.com/gitlab-org/gitlab-foss/issues/49936. # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab-foss/issues/63084 + # https://gitlab.com/gitlab-org/gitlab/issues/29404 data_field :username, :password, :url, :api_url, :jira_issue_transition_id before_update :reset_password @@ -64,7 +64,7 @@ class JiraService < IssueTrackerService url = URI.parse(client_url) { - username: username, + username: username&.strip, password: password, site: URI.join(url, '/').to_s, # Intended to find the root context_path: url.path, @@ -122,9 +122,13 @@ class JiraService < IssueTrackerService end alias_method :original_url, :url - def url - original_url&.chomp('/') + original_url&.delete_suffix('/') + end + + alias_method :original_api_url, :api_url + def api_url + original_api_url&.delete_suffix('/') end def execute(push) @@ -137,10 +141,9 @@ class JiraService < IssueTrackerService return if issue.nil? || has_resolution?(issue) || !jira_issue_transition_id.present? - commit_id = if entity.is_a?(Commit) - entity.id - elsif entity.is_a?(MergeRequest) - entity.diff_head_sha + commit_id = case entity + when Commit then entity.id + when MergeRequest then entity.diff_head_sha end commit_url = build_entity_url(:commit, commit_id) @@ -298,7 +301,7 @@ class JiraService < IssueTrackerService title: title, status: status, icon: { - title: 'GitLab', url16x16: asset_url(Gitlab::Favicon.main, host: gitlab_config.url) + title: 'GitLab', url16x16: asset_url(Gitlab::Favicon.main, host: gitlab_config.base_url) } } } @@ -331,7 +334,6 @@ class JiraService < IssueTrackerService # Handle errors when doing Jira API calls def jira_request yield - rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, URI::InvalidURIError, JIRA::HTTPError, OpenSSL::SSL::SSLError => e @error = e.message log_error("Error sending message", client_url: client_url, error: @error) @@ -339,7 +341,7 @@ class JiraService < IssueTrackerService end def client_url - api_url.present? ? api_url : url + api_url.presence || url end def reset_password? diff --git a/app/models/project_services/packagist_service.rb b/app/models/project_services/packagist_service.rb index 003884bb7ac..35dbedd1341 100644 --- a/app/models/project_services/packagist_service.rb +++ b/app/models/project_services/packagist_service.rb @@ -59,7 +59,7 @@ class PackagistService < Service end def hook_url - base_url = server.present? ? server : 'https://packagist.org' + base_url = server.presence || 'https://packagist.org' "#{base_url}/api/update-package?username=#{username}&apiToken=#{token}" end end diff --git a/app/models/project_services/slash_commands_service.rb b/app/models/project_services/slash_commands_service.rb index 5bfd06476f0..d436176a52c 100644 --- a/app/models/project_services/slash_commands_service.rb +++ b/app/models/project_services/slash_commands_service.rb @@ -33,9 +33,12 @@ class SlashCommandsService < Service return unless valid_token?(params[:token]) chat_user = find_chat_user(params) + user = chat_user&.user + + if user + unless user.can?(:use_slash_commands) + return Gitlab::SlashCommands::Presenters::Access.new.deactivated if user.deactivated? - if chat_user&.user - unless chat_user.user.can?(:use_slash_commands) return Gitlab::SlashCommands::Presenters::Access.new.access_denied(project) end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 218be974218..bb222ac7629 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -54,7 +54,7 @@ class ProjectWiki end def http_url_to_repo - "#{Gitlab.config.gitlab.url}/#{full_path}.git" + @project.http_url_to_repo.sub(%r{git\z}, 'wiki.git') end def wiki_base_path diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 8769d3eb916..1857a59e01c 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -40,6 +40,11 @@ class ProtectedBranch < ApplicationRecord def self.protected_refs(project) project.protected_branches.select(:name) end + + def self.branch_requires_code_owner_approval?(project, branch_name) + # NOOP + # + end end ProtectedBranch.prepend_if_ee('EE::ProtectedBranch') diff --git a/app/models/push_event.rb b/app/models/push_event.rb index 4698df39730..5cab686f20b 100644 --- a/app/models/push_event.rb +++ b/app/models/push_event.rb @@ -26,6 +26,8 @@ class PushEvent < Event delegate :commit_count, to: :push_event_payload alias_method :commits_count, :commit_count + delegate :ref_count, to: :push_event_payload + # Returns events of pushes that either pushed to an existing ref or created a # new one. def self.created_or_pushed @@ -52,7 +54,7 @@ class PushEvent < Event .select(1) .where('merge_requests.source_project_id = events.project_id') .where('merge_requests.source_branch = push_event_payloads.ref') - .where(state: :opened) + .with_state(:opened) # For reasons unknown the use of #eager_load will result in the # "push_event_payload" association not being set. Because of this we're diff --git a/app/models/release.rb b/app/models/release.rb index cd63b4d5fef..5a7bfe2d495 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -14,6 +14,7 @@ class Release < ApplicationRecord has_many :milestone_releases has_many :milestones, through: :milestone_releases + has_one :evidence default_value_for :released_at, allows_nil: false do Time.zone.now @@ -22,13 +23,16 @@ class Release < ApplicationRecord accepts_nested_attributes_for :links, allow_destroy: true validates :description, :project, :tag, presence: true - validates :name, presence: true, on: :create validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") } scope :sorted, -> { order(released_at: :desc) } + scope :with_project_and_namespace, -> { includes(project: :namespace) } delegate :repository, to: :project + after_commit :create_evidence!, on: :create + after_commit :notify_new_release, on: :create + def commit strong_memoize(:commit) do repository.commit(actual_sha) @@ -67,4 +71,14 @@ class Release < ApplicationRecord repository.find_tag(tag) end end + + def create_evidence! + CreateEvidenceWorker.perform_async(self.id) + end + + def notify_new_release + NewReleaseWorker.perform_async(id) + end end + +Release.prepend_if_ee('EE::Release') diff --git a/app/models/repository.rb b/app/models/repository.rb index f084a314392..b9f57169ea5 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -6,6 +6,7 @@ class Repository REF_MERGE_REQUEST = 'merge-requests' REF_KEEP_AROUND = 'keep-around' REF_ENVIRONMENTS = 'environments' + REF_PIPELINES = 'pipelines' ARCHIVE_CACHE_TIME = 60 # Cache archives referred to by a (mutable) ref for 1 minute ARCHIVE_CACHE_TIME_IMMUTABLE = 3600 # Cache archives referred to by an immutable reference for 1 hour @@ -16,7 +17,7 @@ class Repository replace #{REF_ENVIRONMENTS} #{REF_KEEP_AROUND} - #{REF_ENVIRONMENTS} + #{REF_PIPELINES} ].freeze include Gitlab::RepositoryCacheAdapter @@ -133,18 +134,28 @@ class Repository end end - def commits(ref = nil, path: nil, limit: nil, offset: nil, skip_merges: false, after: nil, before: nil, all: nil) + # the opts are: + # - :path + # - :limit + # - :offset + # - :skip_merges + # - :after + # - :before + # - :all + # - :first_parent + def commits(ref = nil, opts = {}) options = { repo: raw_repository, ref: ref, - path: path, - limit: limit, - offset: offset, - after: after, - before: before, - follow: Array(path).length == 1, - skip_merges: skip_merges, - all: all + path: opts[:path], + follow: Array(opts[:path]).length == 1, + limit: opts[:limit], + offset: opts[:offset], + skip_merges: !!opts[:skip_merges], + after: opts[:after], + before: opts[:before], + all: !!opts[:all], + first_parent: !!opts[:first_parent] } commits = Gitlab::Git::Commit.where(options) @@ -239,13 +250,13 @@ class Repository def branch_exists?(branch_name) return false unless raw_repository - branch_names.include?(branch_name) + branch_names_include?(branch_name) end def tag_exists?(tag_name) return false unless raw_repository - tag_names.include?(tag_name) + tag_names_include?(tag_name) end def ref_exists?(ref) @@ -549,15 +560,15 @@ class Repository end delegate :branch_names, to: :raw_repository - cache_method :branch_names, fallback: [] + cache_method_as_redis_set :branch_names, fallback: [] delegate :tag_names, to: :raw_repository - cache_method :tag_names, fallback: [] + cache_method_as_redis_set :tag_names, fallback: [] delegate :branch_count, :tag_count, :has_visible_content?, to: :raw_repository cache_method :branch_count, fallback: 0 cache_method :tag_count, fallback: 0 - cache_method :has_visible_content?, fallback: false + cache_method_asymmetrically :has_visible_content? def avatar # n+1: https://gitlab.com/gitlab-org/gitlab-foss/issues/38327 @@ -1088,6 +1099,8 @@ class Repository raw.create_repository after_create + + true end def blobs_metadata(paths, ref = 'HEAD') diff --git a/app/models/repository_language.rb b/app/models/repository_language.rb index e6867f905e2..e468d716239 100644 --- a/app/models/repository_language.rb +++ b/app/models/repository_language.rb @@ -7,7 +7,7 @@ class RepositoryLanguage < ApplicationRecord default_scope { includes(:programming_language) } validates :project, presence: true - validates :share, inclusion: { in: 0..100, message: "The share of a lanuage is between 0 and 100" } + validates :share, inclusion: { in: 0..100, message: "The share of a language is between 0 and 100" } validates :programming_language, uniqueness: { scope: :project_id } delegate :name, :color, to: :programming_language diff --git a/app/models/service.rb b/app/models/service.rb index 43ed0c7dfaa..305cf7b78a2 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -291,6 +291,12 @@ class Service < ApplicationRecord def self.build_from_template(project_id, template) service = template.dup + + if template.supports_data_fields? + data_fields = template.data_fields.dup + data_fields.service = service + end + service.template = false service.project_id = project_id service.active = false if service.active? && !service.valid? @@ -309,6 +315,11 @@ class Service < ApplicationRecord find_by(template: true) end + # override if needed + def supports_data_fields? + false + end + private def cache_project_has_external_issue_tracker diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 1e84b9fa12e..4010a3e2167 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -14,6 +14,7 @@ class Snippet < ApplicationRecord include Editable include Gitlab::SQL::Pattern include FromUnion + extend ::Gitlab::Utils::Override cache_markdown_field :title, pipeline: :single_line cache_markdown_field :description @@ -32,8 +33,6 @@ class Snippet < ApplicationRecord default_content_html_invalidator || file_name_changed? end - default_value_for(:visibility_level) { Gitlab::CurrentSettings.default_snippet_visibility } - belongs_to :author, class_name: 'User' belongs_to :project @@ -72,7 +71,7 @@ class Snippet < ApplicationRecord end end - def self.only_global_snippets + def self.only_personal_snippets where(project_id: nil) end @@ -138,6 +137,24 @@ class Snippet < ApplicationRecord @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) end + def initialize(attributes = {}) + # We can't use default_value_for because the database has a default + # value of 0 for visibility_level. If someone attempts to create a + # private snippet, default_value_for will assume that the + # visibility_level hasn't changed and will use the application + # setting default, which could be internal or public. + # + # To fix the problem, we assign the actual snippet default if no + # explicit visibility has been initialized. + attributes ||= {} + + unless visibility_attribute_present?(attributes) + attributes[:visibility_level] = Gitlab::CurrentSettings.default_snippet_visibility + end + + super + end + def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{id}" @@ -191,6 +208,12 @@ class Snippet < ApplicationRecord (public? && (title_changed? || content_changed?)) end + # snippers are the biggest sources of spam + override :allow_possible_spam? + def allow_possible_spam? + false + end + def spammable_entity_type 'snippet' end diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index f5d0d6fab3b..9a38b06b2f9 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -27,10 +27,6 @@ module Storage "#{base_dir}/#{disk_hash}" if disk_hash end - def ensure_storage_path_exists - gitlab_shell.add_namespace(repository_storage, base_dir) - end - def rename_repo(old_full_path: nil, new_full_path: nil) true end diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb index 928c773c307..345172cca76 100644 --- a/app/models/storage/legacy_project.rb +++ b/app/models/storage/legacy_project.rb @@ -23,12 +23,6 @@ module Storage project.full_path end - def ensure_storage_path_exists - return unless namespace - - gitlab_shell.add_namespace(repository_storage, base_dir) - end - def rename_repo(old_full_path: nil, new_full_path: nil) old_full_path ||= project.full_path_before_last_save new_full_path ||= project.build_full_path diff --git a/app/models/suggestion.rb b/app/models/suggestion.rb index 22e2f11230d..96ffec90c00 100644 --- a/app/models/suggestion.rb +++ b/app/models/suggestion.rb @@ -41,7 +41,6 @@ class Suggestion < ApplicationRecord !applied? && noteable.opened? && !outdated?(cached: cached) && - note.supports_suggestion? && different_content? && note.active? end diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 8ec90ca25d3..11cbeb60bba 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -23,6 +23,7 @@ class SystemNoteMetadata < ApplicationRecord validates :action, inclusion: { in: :icon_types }, allow_nil: true belongs_to :note + belongs_to :description_version def icon_types ICON_TYPES diff --git a/app/models/todo.rb b/app/models/todo.rb index 6b71845856a..1927b54510e 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -75,13 +75,13 @@ class Todo < ApplicationRecord after_save :keep_around_commit, if: :commit_id class << self - # Returns all todos for the given group and its descendants. + # Returns all todos for the given group ids and their descendants. # - # group - A `Group` to retrieve todos for. + # group_ids - Group Ids to retrieve todos for. # # Returns an `ActiveRecord::Relation`. - def for_group_and_descendants(group) - groups = group.self_and_descendants + def for_group_ids_and_descendants(group_ids) + groups = Group.groups_including_descendants_by(group_ids) from_union([ for_project(Project.for_group(groups)), @@ -144,10 +144,9 @@ class Todo < ApplicationRecord end end - def parent + def resource_parent project end - alias_method :resource_parent, :parent def unmergeable? action == UNMERGEABLE diff --git a/app/models/upload.rb b/app/models/upload.rb index 7560002ada8..8c409641452 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Upload < ApplicationRecord + include Checksummable # Upper limit for foreground checksum processing CHECKSUM_THRESHOLD = 100.megabytes @@ -15,16 +16,12 @@ class Upload < ApplicationRecord scope :with_files_stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) } before_save :calculate_checksum!, if: :foreground_checksummable? - after_commit :schedule_checksum, if: :checksummable? + after_commit :schedule_checksum, if: :needs_checksum? # as the FileUploader is not mounted, the default CarrierWave ActiveRecord # hooks are not executed and the file will not be deleted after_destroy :delete_file!, if: -> { uploader_class <= FileUploader } - def self.hexdigest(path) - Digest::SHA256.file(path).hexdigest - end - class << self ## # FastDestroyAll concerns @@ -53,20 +50,41 @@ class Upload < ApplicationRecord def calculate_checksum! self.checksum = nil - return unless checksummable? + return unless needs_checksum? - self.checksum = Digest::SHA256.file(absolute_path).hexdigest + self.checksum = self.class.hexdigest(absolute_path) end + # Initialize the associated Uploader class with current model + # + # @param [String] mounted_as + # @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute def build_uploader(mounted_as = nil) uploader_class.new(model, mounted_as || mount_point).tap do |uploader| uploader.upload = self + end + end + + # Initialize the associated Uploader class with current model and + # retrieve existing file from the store to a local cache + # + # @param [String] mounted_as + # @return [GitlabUploader] one of the subclasses, defined at the model's uploader attribute + def retrieve_uploader(mounted_as = nil) + build_uploader(mounted_as).tap do |uploader| uploader.retrieve_from_store!(identifier) end end + # This checks for existence of the upload on storage + # + # @return [Boolean] whether upload exists on storage def exist? - exist = File.exist?(absolute_path) + exist = if local? + File.exist?(absolute_path) + else + retrieve_uploader.exists? + end # Help sysadmins find missing upload files if persisted? && !exist @@ -91,18 +109,24 @@ class Upload < ApplicationRecord store == ObjectStorage::Store::LOCAL end + # Returns whether generating checksum is needed + # + # This takes into account whether file exists, if any checksum exists + # or if the storage has checksum generation code implemented + # + # @return [Boolean] whether generating a checksum is needed + def needs_checksum? + checksum.nil? && local? && exist? + end + private def delete_file! - build_uploader.remove! - end - - def checksummable? - checksum.nil? && local? && exist? + retrieve_uploader.remove! end def foreground_checksummable? - checksummable? && size <= CHECKSUM_THRESHOLD + needs_checksum? && size <= CHECKSUM_THRESHOLD end def schedule_checksum @@ -114,7 +138,7 @@ class Upload < ApplicationRecord end def uploader_class - Object.const_get(uploader) + Object.const_get(uploader, false) end def identifier diff --git a/app/models/user.rb b/app/models/user.rb index a69db121a0b..321a4080484 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -59,6 +59,8 @@ class User < ApplicationRecord # Removed in GitLab 12.3. Keep until after 2019-09-22. self.ignored_columns += %i[support_bot] + MINIMUM_INACTIVE_DAYS = 14 + # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour # rubocop: disable CodeReuse/ServiceClass @@ -97,6 +99,7 @@ class User < ApplicationRecord has_many :u2f_registrations, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :chat_names, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :user_synced_attributes_metadata, autosave: true + has_one :aws_role, class_name: 'Aws::Role' # Groups has_many :members @@ -228,6 +231,10 @@ class User < ApplicationRecord # Note: When adding an option, it MUST go on the end of the array. enum project_view: [:readme, :activity, :files] + # User's role + # Note: When adding an option, it MUST go on the end of the array. + enum role: [:software_developer, :development_team_lead, :devops_engineer, :systems_administrator, :security_analyst, :data_analyst, :product_manager, :product_designer, :other], _suffix: true + delegate :path, to: :namespace, allow_nil: true, prefix: true delegate :notes_filter_for, to: :user_preference delegate :set_notes_filter, to: :user_preference @@ -242,18 +249,25 @@ class User < ApplicationRecord state_machine :state, initial: :active do event :block do transition active: :blocked + transition deactivated: :blocked transition ldap_blocked: :blocked end event :ldap_block do transition active: :ldap_blocked + transition deactivated: :ldap_blocked end event :activate do + transition deactivated: :active transition blocked: :active transition ldap_blocked: :active end + event :deactivate do + transition active: :deactivated + end + state :blocked, :ldap_blocked do def blocked? true @@ -284,6 +298,7 @@ class User < ApplicationRecord scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :external, -> { where(external: true) } scope :active, -> { with_state(:active).non_internal } + scope :deactivated, -> { with_state(:deactivated).non_internal } scope :without_projects, -> { joins('LEFT JOIN project_authorizations ON users.id = project_authorizations.user_id').where(project_authorizations: { user_id: nil }) } scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) } scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) } @@ -431,6 +446,8 @@ class User < ApplicationRecord without_projects when 'external' external + when 'deactivated' + deactivated else active end @@ -444,6 +461,7 @@ class User < ApplicationRecord # # Returns an ActiveRecord::Relation. def search(query) + query = query&.delete_prefix('@') return none if query.blank? query = query.downcase @@ -520,7 +538,7 @@ class User < ApplicationRecord # Returns a user for the given SSH key. def find_by_ssh_key_id(key_id) - Key.find_by(id: key_id)&.user + find_by('EXISTS (?)', Key.select(1).where('keys.user_id = users.id').where(id: key_id)) end def find_by_full_path(path, follow_redirects: false) @@ -1303,14 +1321,27 @@ class User < ApplicationRecord notification_group&.notification_email_for(self) || notification_email end - def notification_settings_for(source) + def notification_settings_for(source, inherit: false) if notification_settings.loaded? notification_settings.find do |notification| notification.source_type == source.class.base_class.name && notification.source_id == source.id end else - notification_settings.find_or_initialize_by(source: source) + notification_settings.find_or_initialize_by(source: source) do |ns| + next unless source.is_a?(Group) && inherit + + # If we're here it means we're trying to create a NotificationSetting for a group that doesn't have one. + # Find the closest parent with a notification_setting that's not Global level, or that has an email set. + ancestor_ns = source + .notification_settings(hierarchy_order: :asc) + .where(user: self) + .find_by('level != ? OR notification_email IS NOT NULL', NotificationSetting.levels[:global]) + # Use it to seed the settings + ns.assign_attributes(ancestor_ns&.slice(*NotificationSetting.allowed_fields)) + ns.source = source + ns.user = self + end end end @@ -1529,6 +1560,35 @@ class User < ApplicationRecord todos.find_by(target: target, state: :pending) end + def password_expired? + !!(password_expires_at && password_expires_at < Time.now) + end + + def can_be_deactivated? + active? && no_recent_activity? + end + + def last_active_at + last_activity = last_activity_on&.to_time&.in_time_zone + last_sign_in = current_sign_in_at + + [last_activity, last_sign_in].compact.max + end + + # Below is used for the signup_flow experiment. Should be removed + # when experiment finishes. + # See https://gitlab.com/gitlab-org/growth/engineering/issues/64 + REQUIRES_ROLE_VALUE = 99 + + def role_required? + role_before_type_cast == REQUIRES_ROLE_VALUE + end + + def set_role_required! + update_column(:role, REQUIRES_ROLE_VALUE) + end + # End of signup_flow experiment methods + # @deprecated alias_method :owned_or_masters_groups, :owned_or_maintainers_groups @@ -1678,6 +1738,10 @@ class User < ApplicationRecord ::Group.where(id: developer_groups_hierarchy.select(:id), project_creation_level: project_creation_levels) end + + def no_recent_activity? + last_active_at.to_i <= MINIMUM_INACTIVE_DAYS.days.ago.to_i + end end User.prepend_if_ee('EE::User') diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index cd4c7895587..1fa29e5b933 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -77,11 +77,7 @@ class WikiPage # The escaped URL path of this page. def slug - if @attributes[:slug].present? - @attributes[:slug] - else - wiki.wiki.preview_slug(title, format) - end + @attributes[:slug].presence || wiki.wiki.preview_slug(title, format) end alias_method :to_param, :slug |