diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 14:34:42 +0000 |
| commit | 9f46488805e86b1bc341ea1620b866016c2ce5ed (patch) | |
| tree | f9748c7e287041e37d6da49e0a29c9511dc34768 /app/models | |
| parent | dfc92d081ea0332d69c8aca2f0e745cb48ae5e6d (diff) | |
| download | gitlab-ce-9f46488805e86b1bc341ea1620b866016c2ce5ed.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-0-stable-ee
Diffstat (limited to 'app/models')
138 files changed, 3171 insertions, 1036 deletions
diff --git a/app/models/active_session.rb b/app/models/active_session.rb index 050155398ab..065bd5507be 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -124,7 +124,7 @@ class ActiveSession end end - # Lists the ActiveSession objects for the given session IDs. + # Lists the session Hash objects for the given session IDs. # # session_ids - An array of Rack::Session::SessionId objects # @@ -143,7 +143,7 @@ class ActiveSession end end - # Deserializes an ActiveSession object from Redis. + # Deserializes a session Hash object from Redis. # # raw_session - Raw bytes from Redis # diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb new file mode 100644 index 00000000000..acaf474ecc2 --- /dev/null +++ b/app/models/alert_management/alert.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +module AlertManagement + class Alert < ApplicationRecord + include AtomicInternalId + include ShaAttribute + include Sortable + include Gitlab::SQL::Pattern + + STATUSES = { + triggered: 0, + acknowledged: 1, + resolved: 2, + ignored: 3 + }.freeze + + STATUS_EVENTS = { + triggered: :trigger, + acknowledged: :acknowledge, + resolved: :resolve, + ignored: :ignore + }.freeze + + belongs_to :project + belongs_to :issue, optional: true + has_internal_id :iid, scope: :project, init: ->(s) { s.project.alert_management_alerts.maximum(:iid) } + + self.table_name = 'alert_management_alerts' + + sha_attribute :fingerprint + + HOSTS_MAX_LENGTH = 255 + + validates :title, length: { maximum: 200 }, presence: true + validates :description, length: { maximum: 1_000 } + validates :service, length: { maximum: 100 } + validates :monitoring_tool, length: { maximum: 100 } + validates :project, presence: true + validates :events, presence: true + validates :severity, presence: true + validates :status, presence: true + validates :started_at, presence: true + validates :fingerprint, uniqueness: { scope: :project }, allow_blank: true + validate :hosts_length + + enum severity: { + critical: 0, + high: 1, + medium: 2, + low: 3, + info: 4, + unknown: 5 + } + + state_machine :status, initial: :triggered do + state :triggered, value: STATUSES[:triggered] + + state :acknowledged, value: STATUSES[:acknowledged] + + state :resolved, value: STATUSES[:resolved] do + validates :ended_at, presence: true + end + + state :ignored, value: STATUSES[:ignored] + + state :triggered, :acknowledged, :ignored do + validates :ended_at, absence: true + end + + event :trigger do + transition any => :triggered + end + + event :acknowledge do + transition any => :acknowledged + end + + event :resolve do + transition any => :resolved + end + + event :ignore do + transition any => :ignored + end + + before_transition to: [:triggered, :acknowledged, :ignored] do |alert, _transition| + alert.ended_at = nil + end + + before_transition to: :resolved do |alert, transition| + ended_at = transition.args.first + alert.ended_at = ended_at || Time.current + end + end + + delegate :iid, to: :issue, prefix: true, allow_nil: true + + scope :for_iid, -> (iid) { where(iid: iid) } + scope :for_status, -> (status) { where(status: status) } + scope :for_fingerprint, -> (project, fingerprint) { where(project: project, fingerprint: fingerprint) } + scope :search, -> (query) { fuzzy_search(query, [:title, :description, :monitoring_tool, :service]) } + + scope :order_start_time, -> (sort_order) { order(started_at: sort_order) } + scope :order_end_time, -> (sort_order) { order(ended_at: sort_order) } + scope :order_events_count, -> (sort_order) { order(events: sort_order) } + scope :order_severity, -> (sort_order) { order(severity: sort_order) } + scope :order_status, -> (sort_order) { order(status: sort_order) } + + scope :counts_by_status, -> { group(:status).count } + + def self.sort_by_attribute(method) + case method.to_s + when 'start_time_asc' then order_start_time(:asc) + when 'start_time_desc' then order_start_time(:desc) + when 'end_time_asc' then order_end_time(:asc) + when 'end_time_desc' then order_end_time(:desc) + when 'events_count_asc' then order_events_count(:asc) + when 'events_count_desc' then order_events_count(:desc) + when 'severity_asc' then order_severity(:asc) + when 'severity_desc' then order_severity(:desc) + when 'status_asc' then order_status(:asc) + when 'status_desc' then order_status(:desc) + else + order_by(method) + end + end + + def details + details_payload = payload.except(*attributes.keys) + + Gitlab::Utils::InlineHash.merge_keys(details_payload) + end + + def prometheus? + monitoring_tool == Gitlab::AlertManagement::AlertParams::MONITORING_TOOLS[:prometheus] + end + + private + + def hosts_length + return unless hosts + + errors.add(:hosts, "hosts array is over #{HOSTS_MAX_LENGTH} chars") if hosts.join.length > HOSTS_MAX_LENGTH + end + end +end diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 9da4dfd43b5..00a95070691 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -8,6 +8,7 @@ class Appearance < ApplicationRecord cache_markdown_field :description cache_markdown_field :new_project_guidelines + cache_markdown_field :profile_image_guidelines cache_markdown_field :header_message, pipeline: :broadcast_message cache_markdown_field :footer_message, pipeline: :broadcast_message @@ -15,12 +16,14 @@ class Appearance < ApplicationRecord validates :header_logo, file_size: { maximum: 1.megabyte } validates :message_background_color, allow_blank: true, color: true validates :message_font_color, allow_blank: true, color: true + validates :profile_image_guidelines, length: { maximum: 4096 } validate :single_appearance_row, on: :create default_value_for :title, '' default_value_for :description, '' default_value_for :new_project_guidelines, '' + default_value_for :profile_image_guidelines, '' default_value_for :header_message, '' default_value_for :footer_message, '' default_value_for :message_background_color, '#E75E40' diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 0aa0216558f..b29d6731b08 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -144,7 +144,7 @@ class ApplicationSetting < ApplicationRecord validates :default_artifacts_expire_in, presence: true, duration: true validates :container_expiration_policies_enable_historic_entries, - inclusion: { in: [true, false], message: 'must be a boolean value' } + inclusion: { in: [true, false], message: 'must be a boolean value' } validates :container_registry_token_expire_delay, presence: true, @@ -263,6 +263,8 @@ class ApplicationSetting < ApplicationRecord validates :email_restrictions, untrusted_regexp: true + validates :hashed_storage_enabled, inclusion: { in: [true], message: _("Hashed storage can't be disabled anymore for new projects") } + SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end @@ -345,6 +347,12 @@ class ApplicationSetting < ApplicationRecord presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :issues_create_limit, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + validates :raw_blob_request_limit, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, @@ -412,7 +420,7 @@ class ApplicationSetting < ApplicationRecord # can cause a significant amount of load on Redis, let's cache it in # memory. def self.cache_backend - Gitlab::ThreadMemoryCache.cache_backend + Gitlab::ProcessMemoryCache.cache_backend end def recaptcha_or_login_protection_enabled diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index c96f086684f..221e4d5e0c6 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -43,7 +43,10 @@ module ApplicationSettingImplementation authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand commit_email_hostname: default_commit_email_hostname, container_expiration_policies_enable_historic_entries: false, + container_registry_features: [], container_registry_token_expire_delay: 5, + container_registry_vendor: '', + container_registry_version: '', default_artifacts_expire_in: '30 days', default_branch_protection: Settings.gitlab['default_branch_protection'], default_ci_config_path: nil, @@ -93,7 +96,7 @@ module ApplicationSettingImplementation plantuml_url: nil, polling_interval_multiplier: 1, project_export_enabled: true, - protected_ci_variables: false, + protected_ci_variables: true, push_event_hooks_limit: 3, push_event_activities_limit: 3, raw_blob_request_limit: 300, diff --git a/app/models/blob.rb b/app/models/blob.rb index cdc5838797b..c8df6c7732a 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -86,8 +86,8 @@ class Blob < SimpleDelegator new(blob, container) end - def self.lazy(container, commit_id, path, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) - BatchLoader.for([commit_id, path]).batch(key: container.repository) do |items, loader, args| + def self.lazy(repository, commit_id, path, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) + BatchLoader.for([commit_id, path]).batch(key: repository) do |items, loader, args| args[:key].blobs_at(items, blob_size_limit: blob_size_limit).each do |blob| loader.call([blob.commit_id, blob.path], blob) if blob end @@ -129,7 +129,7 @@ class Blob < SimpleDelegator def external_storage_error? if external_storage == :lfs - !project&.lfs_enabled? + !repository.lfs_enabled? else false end diff --git a/app/models/blob_viewer/dependency_manager.rb b/app/models/blob_viewer/dependency_manager.rb index 711465c7c79..a851f22bfcd 100644 --- a/app/models/blob_viewer/dependency_manager.rb +++ b/app/models/blob_viewer/dependency_manager.rb @@ -32,7 +32,7 @@ module BlobViewer def json_data @json_data ||= begin prepare! - JSON.parse(blob.data) + Gitlab::Json.parse(blob.data) rescue {} end diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 0a536a01f72..856f86201ec 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -105,7 +105,10 @@ class BroadcastMessage < ApplicationRecord def matches_current_path(current_path) return true if current_path.blank? || target_path.blank? - current_path.match(Regexp.escape(target_path).gsub('\\*', '.*')) + escaped = Regexp.escape(target_path).gsub('\\*', '.*') + regexp = Regexp.new "^#{escaped}$", Regexp::IGNORECASE + + regexp.match(current_path) end def flush_redis_cache diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 76882dfcb0d..1e92a47ab49 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -166,6 +166,10 @@ module Ci end end + def dependency_variables + [] + end + private def cross_project_params diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index e515447e394..7f64ea7dd97 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -25,13 +25,16 @@ module Ci RUNNER_FEATURES = { upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? }, - refspecs: -> (build) { build.merge_request_ref? } + refspecs: -> (build) { build.merge_request_ref? }, + artifacts_exclude: -> (build) { build.supports_artifacts_exclude? } }.freeze DEFAULT_RETRIES = { scheduler_failure: 2 }.freeze + DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD' + has_one :deployment, as: :deployable, class_name: 'Deployment' has_one :resource, class_name: 'Ci::Resource', inverse_of: :build has_many :trace_sections, class_name: 'Ci::BuildTraceSection' @@ -87,8 +90,12 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } - scope :with_artifacts_archive, ->() do - where('EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive) + scope :with_downloadable_artifacts, ->() do + where('EXISTS (?)', + Ci::JobArtifact.select(1) + .where('ci_builds.id = ci_job_artifacts.job_id') + .where(file_type: Ci::JobArtifact::DOWNLOADABLE_TYPES) + ) end scope :with_existing_job_artifacts, ->(query) do @@ -130,8 +137,8 @@ module Ci .includes(:metadata, :job_artifacts_metadata) end - scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } - scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } + scope :with_artifacts_not_expired, ->() { with_downloadable_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } + scope :with_expired_artifacts, ->() { with_downloadable_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } scope :scheduled_actions, ->() { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) } @@ -486,8 +493,7 @@ module Ci end def requires_resource? - Feature.enabled?(:ci_resource_group, project, default_enabled: true) && - self.resource_group_id.present? + self.resource_group_id.present? end def has_environment? @@ -530,6 +536,7 @@ module Ci .concat(job_variables) .concat(environment_changed_page_variables) .concat(persisted_environment_variables) + .concat(deploy_freeze_variables) .to_runner_variables end end @@ -585,6 +592,26 @@ module Ci end end + def deploy_freeze_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless freeze_period? + + variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') + end + end + + def freeze_period? + Ci::FreezePeriodStatus.new(project: project).execute + end + + def dependency_variables + return [] if all_dependencies.empty? + + Gitlab::Ci::Variables::Collection.new.concat( + Ci::JobVariable.where(job: all_dependencies).dotenv_source + ) + end + def features { trace_sections: true } end @@ -870,6 +897,14 @@ module Ci end end + def collect_accessibility_reports!(accessibility_report) + each_report(Ci::JobArtifact::ACCESSIBILITY_REPORT_FILE_TYPES) do |file_type, blob| + Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, accessibility_report) + end + + accessibility_report + end + def collect_coverage_reports!(coverage_report) each_report(Ci::JobArtifact::COVERAGE_REPORT_FILE_TYPES) do |file_type, blob| Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, coverage_report) @@ -878,6 +913,14 @@ module Ci coverage_report end + def collect_terraform_reports!(terraform_reports) + each_report(::Ci::JobArtifact::TERRAFORM_REPORT_FILE_TYPES) do |file_type, blob, report_artifact| + ::Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, terraform_reports, artifact: report_artifact) + end + + terraform_reports + end + def report_artifacts job_artifacts.with_reports end @@ -902,6 +945,16 @@ module Ci failure_reason: :data_integrity_failure) end + def supports_artifacts_exclude? + options&.dig(:artifacts, :exclude)&.any? && + Gitlab::Ci::Features.artifacts_exclude_enabled? + end + + def degradation_threshold + var = yaml_variables.find { |v| v[:key] == DEGRADATION_THRESHOLD_VARIABLE_NAME } + var[:value]&.to_i if var + end + private def dependencies diff --git a/app/models/ci/daily_build_group_report_result.rb b/app/models/ci/daily_build_group_report_result.rb new file mode 100644 index 00000000000..3506b27e974 --- /dev/null +++ b/app/models/ci/daily_build_group_report_result.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Ci + class DailyBuildGroupReportResult < ApplicationRecord + extend Gitlab::Ci::Model + + PARAM_TYPES = %w[coverage].freeze + + belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id + belongs_to :project + + def self.upsert_reports(data) + upsert_all(data, unique_by: :index_daily_build_group_report_results_unique_columns) if data.any? + end + + def self.recent_results(attrs, limit: nil) + where(attrs).order(date: :desc, group_name: :asc).limit(limit) + end + end +end diff --git a/app/models/ci/daily_report_result.rb b/app/models/ci/daily_report_result.rb deleted file mode 100644 index 3c1c5f11ed4..00000000000 --- a/app/models/ci/daily_report_result.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Ci - class DailyReportResult < ApplicationRecord - extend Gitlab::Ci::Model - - belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id - belongs_to :project - - # TODO: Refactor this out when BuildReportResult is implemented. - # They both need to share the same enum values for param. - REPORT_PARAMS = { - coverage: 0 - }.freeze - - enum param_type: REPORT_PARAMS - - def self.upsert_reports(data) - upsert_all(data, unique_by: :index_daily_report_results_unique_columns) if data.any? - end - end -end diff --git a/app/models/ci/freeze_period.rb b/app/models/ci/freeze_period.rb new file mode 100644 index 00000000000..bf03b92259a --- /dev/null +++ b/app/models/ci/freeze_period.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class FreezePeriod < ApplicationRecord + include StripAttribute + self.table_name = 'ci_freeze_periods' + + default_scope { order(created_at: :asc) } + + belongs_to :project, inverse_of: :freeze_periods + + strip_attributes :freeze_start, :freeze_end + + validates :freeze_start, cron: true, presence: true + validates :freeze_end, cron: true, presence: true + validates :cron_timezone, cron_freeze_period_timezone: true, presence: true + end +end diff --git a/app/models/ci/freeze_period_status.rb b/app/models/ci/freeze_period_status.rb new file mode 100644 index 00000000000..befa935e750 --- /dev/null +++ b/app/models/ci/freeze_period_status.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Ci + class FreezePeriodStatus + attr_reader :project + + def initialize(project:) + @project = project + end + + def execute + project.freeze_periods.any? { |period| within_freeze_period?(period) } + end + + def within_freeze_period?(period) + # previous_freeze_end, ..., previous_freeze_start, ..., NOW, ..., next_freeze_end, ..., next_freeze_start + # Current time is within a freeze period if + # it falls between a previous freeze start and next freeze end + start_freeze = Gitlab::Ci::CronParser.new(period.freeze_start, period.cron_timezone) + end_freeze = Gitlab::Ci::CronParser.new(period.freeze_end, period.cron_timezone) + + previous_freeze_start = previous_time(start_freeze) + previous_freeze_end = previous_time(end_freeze) + next_freeze_start = next_time(start_freeze) + next_freeze_end = next_time(end_freeze) + + previous_freeze_end < previous_freeze_start && + previous_freeze_start <= time_zone_now && + time_zone_now <= next_freeze_end && + next_freeze_end < next_freeze_start + end + + private + + def previous_time(cron_parser) + cron_parser.previous_time_from(time_zone_now) + end + + def next_time(cron_parser) + cron_parser.next_time_from(time_zone_now) + end + + def time_zone_now + @time_zone_now ||= Time.zone.now + end + end +end diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 15dc1ca8954..4b2081f2977 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -46,7 +46,7 @@ module Ci end def self.fabricate(project, stage) - stage.statuses.ordered.latest + stage.latest_statuses .sort_by(&:sortable_name).group_by(&:group_name) .map do |group_name, grouped_statuses| self.new(project, stage, name: group_name, jobs: grouped_statuses) diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb new file mode 100644 index 00000000000..c674f76d229 --- /dev/null +++ b/app/models/ci/instance_variable.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Ci + class InstanceVariable < ApplicationRecord + extend Gitlab::Ci::Model + include Ci::NewHasVariable + include Ci::Maskable + + alias_attribute :secret_value, :value + + validates :key, uniqueness: { + message: "(%{value}) has already been taken" + } + + scope :unprotected, -> { where(protected: false) } + after_commit { self.class.touch_redis_cache_timestamp } + + class << self + def all_cached + cached_data[:all] + end + + def unprotected_cached + cached_data[:unprotected] + end + + def touch_redis_cache_timestamp(time = Time.current.to_f) + shared_backend.write(:ci_instance_variable_changed_at, time) + end + + private + + def cached_data + fetch_memory_cache(:ci_instance_variable_data) do + all_records = unscoped.all.to_a + + { all: all_records, unprotected: all_records.reject(&:protected?) } + end + end + + def fetch_memory_cache(key, &payload) + cache = process_backend.read(key) + + if cache && !stale_cache?(cache) + cache[:data] + else + store_cache(key, &payload) + end + end + + def stale_cache?(cache_info) + shared_timestamp = shared_backend.read(:ci_instance_variable_changed_at) + return true unless shared_timestamp + + shared_timestamp.to_f > cache_info[:cached_at].to_f + end + + def store_cache(key) + data = yield + time = Time.current.to_f + + process_backend.write(key, data: data, cached_at: time) + touch_redis_cache_timestamp(time) + data + end + + def shared_backend + Rails.cache + end + + def process_backend + Gitlab::ProcessMemoryCache.cache_backend + end + end + end +end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index ef0701b3874..d931428dccd 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -12,7 +12,10 @@ module Ci TEST_REPORT_FILE_TYPES = %w[junit].freeze COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze + ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze NON_ERASABLE_FILE_TYPES = %w[trace].freeze + TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze + UNSUPPORTED_FILE_TYPES = %i[license_management].freeze DEFAULT_FILE_NAMES = { archive: nil, metadata: nil, @@ -20,6 +23,7 @@ module Ci metrics_referee: nil, network_referee: nil, junit: 'junit.xml', + accessibility: 'gl-accessibility.json', codequality: 'gl-code-quality-report.json', sast: 'gl-sast-report.json', dependency_scanning: 'gl-dependency-scanning-report.json', @@ -32,7 +36,8 @@ module Ci lsif: 'lsif.json', dotenv: '.env', cobertura: 'cobertura-coverage.xml', - terraform: 'tfplan.json' + terraform: 'tfplan.json', + cluster_applications: 'gl-cluster-applications.json' }.freeze INTERNAL_TYPES = { @@ -46,13 +51,15 @@ module Ci metrics: :gzip, metrics_referee: :gzip, network_referee: :gzip, - lsif: :gzip, dotenv: :gzip, cobertura: :gzip, + cluster_applications: :gzip, + lsif: :zip, # All these file formats use `raw` as we need to store them uncompressed # for Frontend to fetch the files and do analysis # When they will be only used by backend, they can be `gzipped`. + accessibility: :raw, codequality: :raw, sast: :raw, dependency_scanning: :raw, @@ -64,15 +71,38 @@ module Ci terraform: :raw }.freeze + DOWNLOADABLE_TYPES = %w[ + accessibility + archive + cobertura + codequality + container_scanning + dast + dependency_scanning + dotenv + junit + license_management + license_scanning + lsif + metrics + performance + sast + ].freeze + TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze + # This is required since we cannot add a default to the database + # https://gitlab.com/gitlab-org/gitlab/-/issues/215418 + attribute :locked, :boolean, default: false + belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id mount_uploader :file, JobArtifactUploader validates :file_format, presence: true, unless: :trace?, on: :create - validate :valid_file_format?, unless: :trace?, on: :create + validate :validate_supported_file_format!, on: :create + validate :validate_file_format!, unless: :trace?, on: :create before_save :set_size, if: :file_changed? update_project_statistics project_statistics_name: :build_artifacts_size @@ -82,6 +112,7 @@ module Ci scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } + scope :for_ref, ->(ref, project_id) { joins(job: :pipeline).where(ci_pipelines: { ref: ref, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } scope :with_file_types, -> (file_types) do @@ -98,10 +129,18 @@ module Ci with_file_types(TEST_REPORT_FILE_TYPES) end + scope :accessibility_reports, -> do + with_file_types(ACCESSIBILITY_REPORT_FILE_TYPES) + end + scope :coverage_reports, -> do with_file_types(COVERAGE_REPORT_FILE_TYPES) end + scope :terraform_reports, -> do + with_file_types(TERRAFORM_REPORT_FILE_TYPES) + end + scope :erasable, -> do types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values @@ -109,6 +148,8 @@ module Ci end scope :expired, -> (limit) { where('expire_at < ?', Time.now).limit(limit) } + scope :locked, -> { where(locked: true) } + scope :unlocked, -> { where(locked: [false, nil]) } scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } @@ -133,7 +174,9 @@ module Ci lsif: 15, # LSIF data for code navigation dotenv: 16, cobertura: 17, - terraform: 18 # Transformed json + terraform: 18, # Transformed json + accessibility: 19, + cluster_applications: 20 } enum file_format: { @@ -161,7 +204,15 @@ module Ci raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream }.freeze - def valid_file_format? + def validate_supported_file_format! + return if Feature.disabled?(:drop_license_management_artifact, project, default_enabled: true) + + if UNSUPPORTED_FILE_TYPES.include?(self.file_type&.to_sym) + errors.add(:base, _("File format is no longer supported")) + end + end + + def validate_file_format! unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym errors.add(:base, _('Invalid file format with specified file type')) end diff --git a/app/models/ci/legacy_stage.rb b/app/models/ci/legacy_stage.rb index f156219ea81..250306e2be4 100644 --- a/app/models/ci/legacy_stage.rb +++ b/app/models/ci/legacy_stage.rb @@ -41,6 +41,10 @@ module Ci .fabricate! end + def latest_statuses + statuses.ordered.latest + end + def statuses @statuses ||= pipeline.statuses.where(stage: name) end diff --git a/app/models/ci/persistent_ref.rb b/app/models/ci/persistent_ref.rb index 76139f5d676..91163c85a9e 100644 --- a/app/models/ci/persistent_ref.rb +++ b/app/models/ci/persistent_ref.rb @@ -14,16 +14,12 @@ module Ci delegate :ref_exists?, :create_ref, :delete_refs, to: :repository def exist? - return unless enabled? - ref_exists?(path) rescue false end def create - return unless enabled? - create_ref(sha, path) rescue => e Gitlab::ErrorTracking @@ -31,8 +27,6 @@ module Ci end def delete - return unless enabled? - delete_refs(path) rescue Gitlab::Git::Repository::NoRepository # no-op @@ -44,11 +38,5 @@ module Ci def path "refs/#{Repository::REF_PIPELINES}/#{pipeline.id}" end - - private - - def enabled? - Feature.enabled?(:depend_on_persistent_pipeline_ref, project, default_enabled: true) - end end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 8a3ca2e758c..5db1635f64d 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -82,7 +82,7 @@ module Ci has_one :pipeline_config, class_name: 'Ci::PipelineConfig', inverse_of: :pipeline - has_many :daily_report_results, class_name: 'Ci::DailyReportResult', foreign_key: :last_pipeline_id + has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id accepts_nested_attributes_for :variables, reject_if: :persisted? @@ -115,8 +115,11 @@ module Ci state_machine :status, initial: :created do event :enqueue do - transition [:created, :waiting_for_resource, :preparing, :skipped, :scheduled] => :pending + transition [:created, :manual, :waiting_for_resource, :preparing, :skipped, :scheduled] => :pending transition [:success, :failed, :canceled] => :running + + # this is needed to ensure tests to be covered + transition [:running] => :running end event :request_resource do @@ -194,7 +197,7 @@ module Ci # We wait a little bit to ensure that all BuildFinishedWorkers finish first # because this is where some metrics like code coverage is parsed and stored # in CI build records which the daily build metrics worker relies on. - pipeline.run_after_commit { Ci::DailyReportResultsWorker.perform_in(10.minutes, pipeline.id) } + pipeline.run_after_commit { Ci::DailyBuildGroupReportResultsWorker.perform_in(10.minutes, pipeline.id) } end after_transition do |pipeline, transition| @@ -393,16 +396,18 @@ module Ci false end - ## - # TODO We do not completely switch to persisted stages because of - # race conditions with setting statuses gitlab-foss#23257. - # def ordered_stages - return legacy_stages unless complete? - - if Feature.enabled?('ci_pipeline_persisted_stages', default_enabled: true) + if Feature.enabled?(:ci_atomic_processing, project, default_enabled: false) + # The `Ci::Stage` contains all up-to date data + # as atomic processing updates all data in-bulk + stages + elsif Feature.enabled?(:ci_pipeline_persisted_stages, default_enabled: true) && complete? + # The `Ci::Stage` contains up-to date data only for `completed` pipelines + # this is due to asynchronous processing of pipeline, and stages possibly + # not updated inline with processing of pipeline stages else + # In other cases, we need to calculate stages dynamically legacy_stages end end @@ -440,7 +445,7 @@ module Ci end def legacy_stages - if Feature.enabled?(:ci_composite_status, default_enabled: false) + if Feature.enabled?(:ci_composite_status, project, default_enabled: false) legacy_stages_using_composite_status else legacy_stages_using_sql @@ -681,6 +686,8 @@ module Ci variables.concat(merge_request.predefined_variables) end + variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if has_kubernetes_active? + if external_pull_request_event? && external_pull_request variables.concat(external_pull_request.predefined_variables) end @@ -781,7 +788,7 @@ module Ci end def find_job_with_archive_artifacts(name) - builds.latest.with_artifacts_archive.find_by_name(name) + builds.latest.with_downloadable_artifacts.find_by_name(name) end def latest_builds_with_artifacts @@ -809,6 +816,14 @@ module Ci end end + def accessibility_reports + Gitlab::Ci::Reports::AccessibilityReports.new.tap do |accessibility_reports| + builds.latest.with_reports(Ci::JobArtifact.accessibility_reports).each do |build| + build.collect_accessibility_reports!(accessibility_reports) + end + end + end + def coverage_reports Gitlab::Ci::Reports::CoverageReports.new.tap do |coverage_reports| builds.latest.with_reports(Ci::JobArtifact.coverage_reports).each do |build| @@ -817,6 +832,14 @@ module Ci end end + def terraform_reports + ::Gitlab::Ci::Reports::TerraformReports.new.tap do |terraform_reports| + builds.latest.with_reports(::Ci::JobArtifact.terraform_reports).each do |build| + build.collect_terraform_reports!(terraform_reports) + end + end + end + def has_exposed_artifacts? complete? && builds.latest.with_exposed_artifacts.exists? end @@ -938,6 +961,14 @@ module Ci end end + # Set scheduling type of processables if they were created before scheduling_type + # data was deployed (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22246). + def ensure_scheduling_type! + return unless ::Gitlab::Ci::Features.ensure_scheduling_type_enabled? + + processables.populate_scheduling_type! + end + private def pipeline_data diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index f5785000062..8c9ad343f32 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -6,6 +6,10 @@ module Ci include Importable include StripAttribute include Schedulable + include Limitable + + self.limit_name = 'ci_pipeline_schedules' + self.limit_scope = :project belongs_to :project belongs_to :owner, class_name: 'User' diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index c123bd7c33b..cc00500662d 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -49,7 +49,7 @@ module Ci end validates :type, presence: true - validates :scheduling_type, presence: true, on: :create, if: :validate_scheduling_type? + validates :scheduling_type, presence: true, on: :create, unless: :importing? delegate :merge_request?, :merge_request_ref?, @@ -83,7 +83,7 @@ module Ci # Overriding scheduling_type enum's method for nil `scheduling_type`s def scheduling_type_dag? - super || find_legacy_scheduling_type == :dag + scheduling_type.nil? ? find_legacy_scheduling_type == :dag : super end # scheduling_type column of previous builds/bridges have not been populated, @@ -100,10 +100,12 @@ module Ci end end - private + def ensure_scheduling_type! + # If this has a scheduling_type, it means all processables in the pipeline already have. + return if scheduling_type - def validate_scheduling_type? - !importing? && Feature.enabled?(:validate_scheduling_type_of_processables, project) + pipeline.ensure_scheduling_type! + reset end end end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 93bd42f8734..a316b4718e0 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -13,6 +13,7 @@ module Ci belongs_to :pipeline has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id + has_many :latest_statuses, -> { ordered.latest }, class_name: 'CommitStatus', foreign_key: :stage_id has_many :processables, class_name: 'Ci::Processable', foreign_key: :stage_id has_many :builds, foreign_key: :stage_id has_many :bridges, foreign_key: :stage_id @@ -42,8 +43,7 @@ module Ci state_machine :status, initial: :created do event :enqueue do - transition [:created, :waiting_for_resource, :preparing] => :pending - transition [:success, :failed, :canceled, :skipped] => :running + transition any - [:pending] => :pending end event :request_resource do diff --git a/app/models/clusters/applications/elastic_stack.rb b/app/models/clusters/applications/elastic_stack.rb index afdc1c91c69..0d029aabc3b 100644 --- a/app/models/clusters/applications/elastic_stack.rb +++ b/app/models/clusters/applications/elastic_stack.rb @@ -3,7 +3,7 @@ module Clusters module Applications class ElasticStack < ApplicationRecord - VERSION = '1.9.0' + VERSION = '3.0.0' ELASTICSEARCH_PORT = 9200 @@ -18,7 +18,11 @@ module Clusters default_value_for :version, VERSION def chart - 'stable/elastic-stack' + 'elastic-stack/elastic-stack' + end + + def repository + 'https://charts.gitlab.io' end def install_command @@ -27,7 +31,9 @@ module Clusters version: VERSION, rbac: cluster.platform_kubernetes_rbac?, chart: chart, + repository: repository, files: files, + preinstall: migrate_to_3_script, postinstall: post_install_script ) end @@ -49,7 +55,7 @@ module Clusters strong_memoize(:elasticsearch_client) do next unless kube_client - proxy_url = kube_client.proxy_url('service', 'elastic-stack-elasticsearch-client', ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE) + proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE) Elasticsearch::Client.new(url: proxy_url) do |faraday| # ensures headers containing auth data are appended to original client options @@ -69,23 +75,54 @@ module Clusters end end + def chart_above_v2? + Gem::Version.new(version) >= Gem::Version.new('2.0.0') + end + + def chart_above_v3? + Gem::Version.new(version) >= Gem::Version.new('3.0.0') + end + private + def service_name + chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client' + end + + def pvc_selector + chart_above_v3? ? "app=elastic-stack-elasticsearch-master" : "release=elastic-stack" + end + def post_install_script [ - "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-client:9200" + "timeout -t60 sh /data/helm/elastic-stack/config/wait-for-elasticsearch.sh http://elastic-stack-elasticsearch-master:9200" ] end def post_delete_script [ - Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", "release=elastic-stack") + Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", pvc_selector, "--namespace", Gitlab::Kubernetes::Helm::NAMESPACE) ] end def kube_client cluster&.kubeclient&.core_client end + + def migrate_to_3_script + return [] if !updating? || chart_above_v3? + + # Chart version 3.0.0 moves to our own chart at https://gitlab.com/gitlab-org/charts/elastic-stack + # and is not compatible with pre-existing resources. We first remove them. + [ + Gitlab::Kubernetes::Helm::DeleteCommand.new( + name: 'elastic-stack', + rbac: cluster.platform_kubernetes_rbac?, + files: files + ).delete_command, + Gitlab::Kubernetes::KubectlCmd.delete("pvc", "--selector", "release=elastic-stack", "--namespace", Gitlab::Kubernetes::Helm::NAMESPACE) + ] + end end end end diff --git a/app/models/clusters/applications/fluentd.rb b/app/models/clusters/applications/fluentd.rb index a33b1e39ace..3fd6e870edc 100644 --- a/app/models/clusters/applications/fluentd.rb +++ b/app/models/clusters/applications/fluentd.rb @@ -4,6 +4,7 @@ module Clusters module Applications class Fluentd < ApplicationRecord VERSION = '2.4.0' + CILIUM_CONTAINER_NAME = 'cilium-monitor' self.table_name = 'clusters_applications_fluentd' @@ -18,6 +19,8 @@ module Clusters enum protocol: { tcp: 0, udp: 1 } + validate :has_at_least_one_log_enabled? + def chart 'stable/fluentd' end @@ -39,6 +42,12 @@ module Clusters private + def has_at_least_one_log_enabled? + if !waf_log_enabled && !cilium_log_enabled + errors.add(:base, _("At least one logging option is required to be enabled")) + end + end + def content_values YAML.load_file(chart_values_file).deep_merge!(specification) end @@ -62,7 +71,7 @@ module Clusters program fluentd hostname ${kubernetes_host} protocol #{protocol} - packet_size 65535 + packet_size 131072 <buffer kubernetes_host> </buffer> <format> @@ -85,7 +94,7 @@ module Clusters <source> @type tail @id in_tail_container_logs - path /var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log + path #{path_to_logs} pos_file /var/log/fluentd-containers.log.pos tag kubernetes.* read_from_head true @@ -96,6 +105,13 @@ module Clusters </source> EOF end + + def path_to_logs + path = [] + path << "/var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" if waf_log_enabled + path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled + path.join(',') + end end end end diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 5985e08d73e..dd354198910 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -17,6 +17,7 @@ module Clusters include ::Clusters::Concerns::ApplicationVersion include ::Clusters::Concerns::ApplicationData include AfterCommitQueue + include UsageStatistics default_value_for :ingress_type, :nginx default_value_for :modsecurity_enabled, true @@ -29,6 +30,10 @@ module Clusters enum modsecurity_mode: { logging: 0, blocking: 1 } + scope :modsecurity_not_installed, -> { where(modsecurity_enabled: nil) } + scope :modsecurity_enabled, -> { where(modsecurity_enabled: true) } + scope :modsecurity_disabled, -> { where(modsecurity_enabled: false) } + FETCH_IP_ADDRESS_DELAY = 30.seconds state_machine :status do @@ -98,7 +103,7 @@ module Clusters "args" => [ "/bin/sh", "-c", - "tail -f /var/log/modsec/audit.log" + "tail -F /var/log/modsec/audit.log" ], "volumeMounts" => [ { diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index 42fa4a6f179..056ea355de6 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -5,7 +5,7 @@ require 'securerandom' module Clusters module Applications class Jupyter < ApplicationRecord - VERSION = '0.9.0-beta.2' + VERSION = '0.9.0' self.table_name = 'clusters_applications_jupyter' diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb index 1f90318f845..3047da12dd9 100644 --- a/app/models/clusters/applications/knative.rb +++ b/app/models/clusters/applications/knative.rb @@ -4,8 +4,8 @@ module Clusters module Applications class Knative < ApplicationRecord VERSION = '0.9.0' - REPOSITORY = 'https://storage.googleapis.com/triggermesh-charts' - METRICS_CONFIG = 'https://storage.googleapis.com/triggermesh-charts/istio-metrics.yaml' + REPOSITORY = 'https://charts.gitlab.io' + METRICS_CONFIG = 'https://gitlab.com/gitlab-org/charts/knative/-/raw/v0.9.0/vendor/istio-metrics.yml' FETCH_IP_ADDRESS_DELAY = 30.seconds API_GROUPS_PATH = 'config/knative/api_groups.yml' diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 7d67e258991..a861126908f 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.15.0' + VERSION = '0.16.1' self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 430a9b3c43e..83f558af1a1 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -26,6 +26,8 @@ module Clusters KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN' APPLICATIONS_ASSOCIATIONS = APPLICATIONS.values.map(&:association_name).freeze + self.reactive_cache_work_type = :external_dependency + belongs_to :user belongs_to :management_project, class_name: '::Project', optional: true @@ -33,6 +35,7 @@ module Clusters has_many :projects, through: :cluster_projects, class_name: '::Project' has_one :cluster_project, -> { order(id: :desc) }, class_name: 'Clusters::Project' has_many :deployment_clusters + has_many :deployments, inverse_of: :cluster has_many :cluster_groups, class_name: 'Clusters::Group' has_many :groups, through: :cluster_groups, class_name: '::Group' @@ -203,10 +206,16 @@ module Clusters end end + def nodes + with_reactive_cache do |data| + data[:nodes] + end + end + def calculate_reactive_cache return unless enabled? - { connection_status: retrieve_connection_status } + { connection_status: retrieve_connection_status, nodes: retrieve_nodes } end def persisted_applications @@ -214,11 +223,19 @@ module Clusters end def applications - APPLICATIONS_ASSOCIATIONS.map do |association_name| - public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend + APPLICATIONS.each_value.map do |application_class| + find_or_build_application(application_class) end end + def find_or_build_application(application_class) + raise ArgumentError, "#{application_class} is not in APPLICATIONS" unless APPLICATIONS.value?(application_class) + + association_name = application_class.association_name + + public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend + end + def provider if gcp? provider_gcp @@ -345,32 +362,55 @@ module Clusters end def retrieve_connection_status - kubeclient.core_client.discover - rescue *Gitlab::Kubernetes::Errors::CONNECTION - :unreachable - rescue *Gitlab::Kubernetes::Errors::AUTHENTICATION - :authentication_failure - rescue Kubeclient::HttpError => e - kubeclient_error_status(e.message) - rescue => e - Gitlab::ErrorTracking.track_exception(e, cluster_id: id) - - :unknown_failure - else - :connected - end - - # KubeClient uses the same error class - # For connection errors (eg. timeout) and - # for Kubernetes errors. - def kubeclient_error_status(message) - if message&.match?(/timed out|timeout/i) - :unreachable - else - :authentication_failure + result = ::Gitlab::Kubernetes::KubeClient.graceful_request(id) { kubeclient.core_client.discover } + result[:status] + end + + def retrieve_nodes + result = ::Gitlab::Kubernetes::KubeClient.graceful_request(id) { kubeclient.get_nodes } + cluster_nodes = result[:response].to_a + + result = ::Gitlab::Kubernetes::KubeClient.graceful_request(id) { kubeclient.metrics_client.get_nodes } + nodes_metrics = result[:response].to_a + + cluster_nodes.inject([]) do |memo, node| + sliced_node = filter_relevant_node_attributes(node) + + matched_node_metric = nodes_metrics.find { |node_metric| node_metric.metadata.name == node.metadata.name } + + sliced_node_metrics = matched_node_metric ? filter_relevant_node_metrics_attributes(matched_node_metric) : {} + + memo << sliced_node.merge(sliced_node_metrics) end end + def filter_relevant_node_attributes(node) + { + 'metadata' => { + 'name' => node.metadata.name + }, + 'status' => { + 'capacity' => { + 'cpu' => node.status.capacity.cpu, + 'memory' => node.status.capacity.memory + }, + 'allocatable' => { + 'cpu' => node.status.allocatable.cpu, + 'memory' => node.status.allocatable.memory + } + } + } + end + + def filter_relevant_node_metrics_attributes(node_metrics) + { + 'usage' => { + 'cpu' => node_metrics.usage.cpu, + 'memory' => node_metrics.usage.memory + } + } + end + # To keep backward compatibility with AUTO_DEVOPS_DOMAIN # environment variable, we need to ensure KUBE_INGRESS_BASE_DOMAIN # is set if AUTO_DEVOPS_DOMAIN is set on any of the following options: diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb index 14237439a8d..0b915126f8a 100644 --- a/app/models/clusters/concerns/application_status.rb +++ b/app/models/clusters/concerns/application_status.rb @@ -27,6 +27,7 @@ module Clusters state :update_errored, value: 6 state :uninstalling, value: 7 state :uninstall_errored, value: 8 + state :uninstalled, value: 10 # Used for applications that are pre-installed by the cluster, # e.g. Knative in GCP Cloud Run enabled clusters @@ -35,6 +36,14 @@ module Clusters # and no exit transitions. state :pre_installed, value: 9 + event :make_externally_installed do + transition any => :installed + end + + event :make_externally_uninstalled do + transition any => :uninstalled + end + event :make_scheduled do transition [:installable, :errored, :installed, :updated, :update_errored, :uninstall_errored] => :scheduled end diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 046f131b041..7e99f128dad 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -7,8 +7,6 @@ class CommitStatus < ApplicationRecord include Presentable include EnumWithNil - prepend_if_ee('::EE::CommitStatus') # rubocop: disable Cop/InjectEnterpriseEditionModule - self.table_name = 'ci_builds' belongs_to :user @@ -267,8 +265,16 @@ class CommitStatus < ApplicationRecord end end + def recoverable? + failed? && !unrecoverable_failure? + end + private + def unrecoverable_failure? + script_failure? || missing_dependency_failure? || archived_failure? || scheduler_failure? || data_integrity_failure? + end + def schedule_stage_and_pipeline_update if Feature.enabled?(:ci_atomic_processing, project) # Atomic Processing requires only single Worker @@ -284,3 +290,5 @@ class CommitStatus < ApplicationRecord end end end + +CommitStatus.prepend_if_ee('::EE::CommitStatus') diff --git a/app/models/concerns/async_devise_email.rb b/app/models/concerns/async_devise_email.rb new file mode 100644 index 00000000000..38c99dc7e71 --- /dev/null +++ b/app/models/concerns/async_devise_email.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module AsyncDeviseEmail + extend ActiveSupport::Concern + + private + + # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration + def send_devise_notification(notification, *args) + return true unless can?(:receive_notifications) + + devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend + end +end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index 0f2a389f0a3..896f0916d8c 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -14,32 +14,29 @@ module Awardable class_methods do def awarded(user, name = nil) - sql = <<~EOL - EXISTS ( - SELECT TRUE - FROM award_emoji - WHERE user_id = :user_id AND - #{"name = :name AND" if name.present?} - awardable_type = :awardable_type AND - awardable_id = #{self.arel_table.name}.id - ) - EOL + award_emoji_table = Arel::Table.new('award_emoji') + inner_query = award_emoji_table + .project('true') + .where(award_emoji_table[:user_id].eq(user.id)) + .where(award_emoji_table[:awardable_type].eq(self.name)) + .where(award_emoji_table[:awardable_id].eq(self.arel_table[:id])) + + inner_query = inner_query.where(award_emoji_table[:name].eq(name)) if name.present? - where(sql, user_id: user.id, name: name, awardable_type: self.name) + where(inner_query.exists) end - def not_awarded(user) - sql = <<~EOL - NOT EXISTS ( - SELECT TRUE - FROM award_emoji - WHERE user_id = :user_id AND - awardable_type = :awardable_type AND - awardable_id = #{self.arel_table.name}.id - ) - EOL + def not_awarded(user, name = nil) + award_emoji_table = Arel::Table.new('award_emoji') + inner_query = award_emoji_table + .project('true') + .where(award_emoji_table[:user_id].eq(user.id)) + .where(award_emoji_table[:awardable_type].eq(self.name)) + .where(award_emoji_table[:awardable_id].eq(self.arel_table[:id])) + + inner_query = inner_query.where(award_emoji_table[:name].eq(name)) if name.present? - where(sql, user_id: user.id, awardable_type: self.name) + where(inner_query.exists.not) end def order_upvotes_desc @@ -77,7 +74,7 @@ module Awardable # By default we always load award_emoji user association awards = award_emoji.group_by(&:name) - if with_thumbs + if with_thumbs && (!project || project.show_default_award_emojis?) awards[AwardEmoji::UPVOTE_NAME] ||= [] awards[AwardEmoji::DOWNVOTE_NAME] ||= [] end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index cc13f279c4d..e4e0f55d5f4 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -161,7 +161,6 @@ module CacheMarkdownField define_method(invalidation_method) do changed_fields = changed_attributes.keys invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY] - invalidations.delete(markdown_field.to_s) if changed_fields.include?("#{markdown_field}_html") !invalidations.empty? || !cached_html_up_to_date?(markdown_field) end end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index 5ff537a7837..ccd90ea5900 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -18,6 +18,8 @@ module Ci variables.concat(deployment_variables(environment: environment)) variables.concat(yaml_variables) variables.concat(user_variables) + variables.concat(dependency_variables) if Feature.enabled?(:ci_dependency_variables, project) + variables.concat(secret_instance_variables) variables.concat(secret_group_variables) variables.concat(secret_project_variables(environment: environment)) variables.concat(trigger_request.user_variables) if trigger_request @@ -81,6 +83,12 @@ module Ci ) end + def secret_instance_variables + return [] unless ::Feature.enabled?(:ci_instance_level_variables, project, default_enabled: true) + + project.ci_instance_variables_for(ref: git_ref) + end + def secret_group_variables return [] unless project.group diff --git a/app/models/concerns/diff_positionable_note.rb b/app/models/concerns/diff_positionable_note.rb index 6484a3157b1..cea3c7d119c 100644 --- a/app/models/concerns/diff_positionable_note.rb +++ b/app/models/concerns/diff_positionable_note.rb @@ -17,12 +17,14 @@ module DiffPositionableNote %i(original_position position change_position).each do |meth| define_method "#{meth}=" do |new_position| if new_position.is_a?(String) - new_position = JSON.parse(new_position) rescue nil + new_position = Gitlab::Json.parse(new_position) rescue nil end if new_position.is_a?(Hash) new_position = new_position.with_indifferent_access new_position = Gitlab::Diff::Position.new(new_position) + elsif !new_position.is_a?(Gitlab::Diff::Position) + new_position = nil end return if new_position == read_attribute(meth) diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb index af7afd6604a..29d31b8bb4f 100644 --- a/app/models/concerns/has_repository.rb +++ b/app/models/concerns/has_repository.rb @@ -9,7 +9,6 @@ # needs any special behavior. module HasRepository extend ActiveSupport::Concern - include AfterCommitQueue include Referable include Gitlab::ShellAdapter include Gitlab::Utils::StrongMemoize diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb new file mode 100644 index 00000000000..8a238dc736c --- /dev/null +++ b/app/models/concerns/has_user_type.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module HasUserType + extend ActiveSupport::Concern + + USER_TYPES = { + human: nil, + support_bot: 1, + alert_bot: 2, + visual_review_bot: 3, + service_user: 4, + ghost: 5, + project_bot: 6, + migration_bot: 7 + }.with_indifferent_access.freeze + + BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot].freeze + NON_INTERNAL_USER_TYPES = %w[human project_bot service_user].freeze + INTERNAL_USER_TYPES = (USER_TYPES.keys - NON_INTERNAL_USER_TYPES).freeze + + included do + scope :humans, -> { where(user_type: :human) } + scope :bots, -> { where(user_type: BOT_USER_TYPES) } + scope :bots_without_project_bot, -> { where(user_type: BOT_USER_TYPES - ['project_bot']) } + scope :non_internal, -> { humans.or(where(user_type: NON_INTERNAL_USER_TYPES)) } + scope :without_ghosts, -> { humans.or(where.not(user_type: :ghost)) } + scope :without_project_bot, -> { humans.or(where.not(user_type: :project_bot)) } + + enum user_type: USER_TYPES + + def human? + super || user_type.nil? + end + end + + def bot? + BOT_USER_TYPES.include?(user_type) + end + + # The explicit check for project_bot will be removed with Bot Categorization + # Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/213945 + def internal? + ghost? || (bot? && !project_bot?) + end +end diff --git a/app/models/concerns/has_wiki.rb b/app/models/concerns/has_wiki.rb new file mode 100644 index 00000000000..4dd72216e77 --- /dev/null +++ b/app/models/concerns/has_wiki.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module HasWiki + extend ActiveSupport::Concern + + included do + validate :check_wiki_path_conflict + end + + def create_wiki + wiki.wiki + true + rescue Wiki::CouldNotCreateWikiError + errors.add(:base, _('Failed to create wiki')) + false + end + + def wiki + strong_memoize(:wiki) do + Wiki.for_container(self, self.owner) + end + end + + def wiki_repository_exists? + wiki.repository_exists? + end + + def after_wiki_activity + true + end + + private + + def check_wiki_path_conflict + return if path.blank? + + path_to_check = path.ends_with?('.wiki') ? path.chomp('.wiki') : "#{path}.wiki" + + if Project.in_namespace(parent_id).where(path: path_to_check).exists? || + GroupsFinder.new(nil, parent: parent_id).execute.where(path: path_to_check).exists? + errors.add(:name, _('has already been taken')) + end + end +end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 37f2209b9d2..a1b14dca4ac 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -115,9 +115,31 @@ module Issuable end # rubocop:enable GitlabSecurity/SqlInjection + scope :not_assigned_to, ->(users) do + assignees_table = Arel::Table.new("#{to_ability_name}_assignees") + sql = assignees_table.project('true') + .where(assignees_table[:user_id].in(users)) + .where(Arel::Nodes::SqlLiteral.new("#{to_ability_name}_id = #{to_ability_name}s.id")) + where(sql.exists.not) + end + + scope :without_particular_labels, ->(label_names) do + labels_table = Label.arel_table + label_links_table = LabelLink.arel_table + issuables_table = klass.arel_table + inner_query = label_links_table.project('true') + .join(labels_table, Arel::Nodes::InnerJoin).on(labels_table[:id].eq(label_links_table[:label_id])) + .where(label_links_table[:target_type].eq(name) + .and(label_links_table[:target_id].eq(issuables_table[:id])) + .and(labels_table[:title].in(label_names))) + .exists.not + + where(inner_query) + end + scope :without_label, -> { joins("LEFT OUTER JOIN label_links ON label_links.target_type = '#{name}' AND label_links.target_id = #{table_name}.id").where(label_links: { id: nil }) } scope :with_label_ids, ->(label_ids) { joins(:label_links).where(label_links: { label_id: label_ids }) } - scope :any_label, -> { joins(:label_links).group(:id) } + scope :any_label, -> { joins(:label_links).distinct } scope :join_project, -> { joins(:project) } scope :inc_notes_with_associations, -> { includes(notes: [:project, :author, :award_emoji]) } scope :references_project, -> { references(:project) } @@ -286,9 +308,8 @@ module Issuable .reorder(Gitlab::Database.nulls_last_order('highest_priority', direction)) end - def with_label(title, sort = nil, not_query: false) - multiple_labels = title.is_a?(Array) && title.size > 1 - if multiple_labels && !not_query + def with_label(title, sort = nil) + if title.is_a?(Array) && title.size > 1 joins(:labels).where(labels: { title: title }).group(*grouping_columns(sort)).having("COUNT(DISTINCT labels.title) = #{title.size}") else joins(:labels).where(labels: { title: title }) diff --git a/app/models/concerns/issue_resource_event.rb b/app/models/concerns/issue_resource_event.rb new file mode 100644 index 00000000000..1c24032dbbb --- /dev/null +++ b/app/models/concerns/issue_resource_event.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module IssueResourceEvent + extend ActiveSupport::Concern + + included do + belongs_to :issue + + scope :by_issue, ->(issue) { where(issue_id: issue.id) } + + scope :by_issue_ids_and_created_at_earlier_or_equal_to, ->(issue_ids, time) { where(issue_id: issue_ids).where('created_at <= ?', time) } + end +end diff --git a/app/models/concerns/limitable.rb b/app/models/concerns/limitable.rb new file mode 100644 index 00000000000..f320f54bb82 --- /dev/null +++ b/app/models/concerns/limitable.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Limitable + extend ActiveSupport::Concern + + included do + class_attribute :limit_scope + class_attribute :limit_name + self.limit_name = self.name.demodulize.tableize + + validate :validate_plan_limit_not_exceeded, on: :create + end + + private + + def validate_plan_limit_not_exceeded + scope_relation = self.public_send(limit_scope) # rubocop:disable GitlabSecurity/PublicSend + return unless scope_relation + + relation = self.class.where(limit_scope => scope_relation) + + if scope_relation.actual_limits.exceeded?(limit_name, relation) + errors.add(:base, _("Maximum number of %{name} (%{count}) exceeded") % + { name: limit_name.humanize(capitalize: false), count: scope_relation.actual_limits.public_send(limit_name) }) # rubocop:disable GitlabSecurity/PublicSend + end + end +end diff --git a/app/models/concerns/merge_request_resource_event.rb b/app/models/concerns/merge_request_resource_event.rb new file mode 100644 index 00000000000..7fb7fb4ec62 --- /dev/null +++ b/app/models/concerns/merge_request_resource_event.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module MergeRequestResourceEvent + extend ActiveSupport::Concern + + included do + belongs_to :merge_request + + scope :by_merge_request, ->(merge_request) { where(merge_request_id: merge_request.id) } + end +end diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb index 3ffb32f94fc..8f8494a9678 100644 --- a/app/models/concerns/milestoneable.rb +++ b/app/models/concerns/milestoneable.rb @@ -17,8 +17,10 @@ module Milestoneable scope :of_milestones, ->(ids) { where(milestone_id: ids) } scope :any_milestone, -> { where('milestone_id IS NOT NULL') } scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) } + scope :without_particular_milestone, ->(title) { left_outer_joins(:milestone).where("milestones.title != ? OR milestone_id IS NULL", title) } scope :any_release, -> { joins_milestone_releases } scope :with_release, -> (tag, project_id) { joins_milestone_releases.where( milestones: { releases: { tag: tag, project_id: project_id } } ) } + scope :without_particular_release, -> (tag, project_id) { joins_milestone_releases.where.not( milestones: { releases: { tag: tag, project_id: project_id } } ) } scope :left_joins_milestones, -> { joins("LEFT OUTER JOIN milestones ON #{table_name}.milestone_id = milestones.id") } scope :order_milestone_due_desc, -> { left_joins_milestones.reorder(Arel.sql('milestones.due_date IS NULL, milestones.id IS NULL, milestones.due_date DESC')) } diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index a7f1fb66a88..933a0b167e2 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -17,7 +17,7 @@ module Noteable # `Noteable` class names that support resolvable notes. def resolvable_types - %w(MergeRequest) + %w(MergeRequest DesignManagement::Design) end end @@ -138,15 +138,25 @@ module Noteable end def note_etag_key + return Gitlab::Routing.url_helpers.designs_project_issue_path(project, issue, { vueroute: filename }) if self.is_a?(DesignManagement::Design) + Gitlab::Routing.url_helpers.project_noteable_notes_path( project, target_type: self.class.name.underscore, target_id: id ) end + + def after_note_created(_note) + # no-op + end + + def after_note_destroyed(_note) + # no-op + end end Noteable.extend(Noteable::ClassMethods) -Noteable::ClassMethods.prepend_if_ee('EE::Noteable::ClassMethods') # rubocop: disable Cop/InjectEnterpriseEditionModule +Noteable::ClassMethods.prepend_if_ee('EE::Noteable::ClassMethods') Noteable.prepend_if_ee('EE::Noteable') diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index abc41a1c476..761a151a474 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -9,6 +9,7 @@ module PrometheusAdapter self.reactive_cache_lease_timeout = 30.seconds self.reactive_cache_refresh_interval = 30.seconds self.reactive_cache_lifetime = 1.minute + self.reactive_cache_work_type = :external_dependency def prometheus_client raise NotImplementedError diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index 7373f006d64..d1e3d9b2aff 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -50,8 +50,8 @@ module ProtectedRefAccess end end -ProtectedRefAccess.include_if_ee('EE::ProtectedRefAccess::Scopes') # rubocop: disable Cop/InjectEnterpriseEditionModule -ProtectedRefAccess.prepend_if_ee('EE::ProtectedRefAccess') # rubocop: disable Cop/InjectEnterpriseEditionModule +ProtectedRefAccess.include_if_ee('EE::ProtectedRefAccess::Scopes') +ProtectedRefAccess.prepend_if_ee('EE::ProtectedRefAccess') # When using `prepend` (or `include` for that matter), the `ClassMethods` # constants are not merged. This means that `class_methods` in diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb index 4b472cfdf45..d294563139c 100644 --- a/app/models/concerns/reactive_caching.rb +++ b/app/models/concerns/reactive_caching.rb @@ -8,6 +8,11 @@ module ReactiveCaching InvalidateReactiveCache = Class.new(StandardError) ExceededReactiveCacheLimit = Class.new(StandardError) + WORK_TYPE = { + default: ReactiveCachingWorker, + external_dependency: ExternalServiceReactiveCachingWorker + }.freeze + included do extend ActiveModel::Naming @@ -16,6 +21,7 @@ module ReactiveCaching class_attribute :reactive_cache_refresh_interval class_attribute :reactive_cache_lifetime class_attribute :reactive_cache_hard_limit + class_attribute :reactive_cache_work_type class_attribute :reactive_cache_worker_finder # defaults @@ -24,6 +30,7 @@ module ReactiveCaching self.reactive_cache_refresh_interval = 1.minute self.reactive_cache_lifetime = 10.minutes self.reactive_cache_hard_limit = 1.megabyte + self.reactive_cache_work_type = :default self.reactive_cache_worker_finder = ->(id, *_args) do find_by(primary_key => id) end @@ -112,7 +119,7 @@ module ReactiveCaching def refresh_reactive_cache!(*args) clear_reactive_cache!(*args) keep_alive_reactive_cache!(*args) - ReactiveCachingWorker.perform_async(self.class, id, *args) + worker_class.perform_async(self.class, id, *args) end def keep_alive_reactive_cache!(*args) @@ -145,7 +152,11 @@ module ReactiveCaching def enqueuing_update(*args) yield - ReactiveCachingWorker.perform_in(self.class.reactive_cache_refresh_interval, self.class, id, *args) + worker_class.perform_in(self.class.reactive_cache_refresh_interval, self.class, id, *args) + end + + def worker_class + WORK_TYPE.fetch(self.class.reactive_cache_work_type.to_sym) end def check_exceeded_reactive_cache_limit!(data) diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index 4bb4ffe2a8e..2d4ed51ce3b 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -26,7 +26,7 @@ module RedisCacheable end def cache_attributes(values) - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.set(cache_attribute_key, values.to_json, ex: CACHED_ATTRIBUTES_EXPIRY_TIME) end @@ -41,9 +41,9 @@ module RedisCacheable def cached_attributes strong_memoize(:cached_attributes) do - Gitlab::Redis::SharedState.with do |redis| + Gitlab::Redis::Cache.with do |redis| data = redis.get(cache_attribute_key) - JSON.parse(data, symbolize_names: true) if data + Gitlab::Json.parse(data, symbolize_names: true) if data end end end diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index 4fbb5dcb649..9cd1a22b203 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -13,9 +13,13 @@ module Spammable has_one :user_agent_detail, as: :subject, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent attr_accessor :spam + attr_accessor :needs_recaptcha attr_accessor :spam_log + alias_method :spam?, :spam + alias_method :needs_recaptcha?, :needs_recaptcha + # if spam errors are added before validation, they will be wiped after_validation :invalidate_if_spam, on: [:create, :update] cattr_accessor :spammable_attrs, instance_accessor: false do @@ -38,24 +42,35 @@ module Spammable end def needs_recaptcha! - self.errors.add(:base, "Your #{spammable_entity_type} has been recognized as spam. "\ - "Please, change the content or solve the reCAPTCHA to proceed.") + self.needs_recaptcha = true end - def unrecoverable_spam_error! - self.errors.add(:base, "Your #{spammable_entity_type} has been recognized as spam and has been discarded.") + def spam! + self.spam = true end - def invalidate_if_spam - return unless spam? + def clear_spam_flags! + self.spam = false + self.needs_recaptcha = false + end - if Gitlab::Recaptcha.enabled? - needs_recaptcha! - else + def invalidate_if_spam + if needs_recaptcha? && Gitlab::Recaptcha.enabled? + recaptcha_error! + elsif needs_recaptcha? || spam? unrecoverable_spam_error! end end + def recaptcha_error! + self.errors.add(:base, "Your #{spammable_entity_type} has been recognized as spam. "\ + "Please, change the content or solve the reCAPTCHA to proceed.") + end + + def unrecoverable_spam_error! + self.errors.add(:base, "Your #{spammable_entity_type} has been recognized as spam and has been discarded.") + end + def spammable_entity_type self.class.name.underscore end diff --git a/app/models/concerns/state_eventable.rb b/app/models/concerns/state_eventable.rb new file mode 100644 index 00000000000..68129798543 --- /dev/null +++ b/app/models/concerns/state_eventable.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module StateEventable + extend ActiveSupport::Concern + + included do + has_many :resource_state_events + end +end diff --git a/app/models/concerns/storage/legacy_project_wiki.rb b/app/models/concerns/storage/legacy_project_wiki.rb deleted file mode 100644 index a377fa1e5de..00000000000 --- a/app/models/concerns/storage/legacy_project_wiki.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Storage - module LegacyProjectWiki - extend ActiveSupport::Concern - - def disk_path - project.disk_path + '.wiki' - end - end -end diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb new file mode 100644 index 00000000000..d29e6a01c56 --- /dev/null +++ b/app/models/concerns/timebox.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +module Timebox + extend ActiveSupport::Concern + + include AtomicInternalId + include CacheMarkdownField + include Gitlab::SQL::Pattern + include IidRoutes + include StripAttribute + + TimeboxStruct = Struct.new(:title, :name, :id) do + # Ensure these models match the interface required for exporting + def serializable_hash(_opts = {}) + { title: title, name: name, id: id } + end + end + + # Represents a "No Timebox" state used for filtering Issues and Merge + # Requests that have no timeboxes assigned. + None = TimeboxStruct.new('No Timebox', 'No Timebox', 0) + Any = TimeboxStruct.new('Any Timebox', '', -1) + Upcoming = TimeboxStruct.new('Upcoming', '#upcoming', -2) + Started = TimeboxStruct.new('Started', '#started', -3) + + included do + # Defines the same constants above, but inside the including class. + const_set :None, TimeboxStruct.new("No #{self.name}", "No #{self.name}", 0) + const_set :Any, TimeboxStruct.new("Any #{self.name}", '', -1) + const_set :Upcoming, TimeboxStruct.new('Upcoming', '#upcoming', -2) + const_set :Started, TimeboxStruct.new('Started', '#started', -3) + + alias_method :timebox_id, :id + + validates :group, presence: true, unless: :project + validates :project, presence: true, unless: :group + validates :title, presence: true + + validate :uniqueness_of_title, if: :title_changed? + validate :timebox_type_check + validate :start_date_should_be_less_than_due_date, if: proc { |m| m.start_date.present? && m.due_date.present? } + validate :dates_within_4_digits + + cache_markdown_field :title, pipeline: :single_line + cache_markdown_field :description + + belongs_to :project + belongs_to :group + + has_many :issues + has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues + has_many :merge_requests + + scope :of_projects, ->(ids) { where(project_id: ids) } + scope :of_groups, ->(ids) { where(group_id: ids) } + scope :closed, -> { with_state(:closed) } + scope :for_projects, -> { where(group: nil).includes(:project) } + scope :with_title, -> (title) { where(title: title) } + + scope :for_projects_and_groups, -> (projects, groups) do + projects = projects.compact if projects.is_a? Array + projects = [] if projects.nil? + + groups = groups.compact if groups.is_a? Array + groups = [] if groups.nil? + + where(project_id: projects).or(where(group_id: groups)) + end + + scope :within_timeframe, -> (start_date, end_date) do + where('start_date is not NULL or due_date is not NULL') + .where('start_date is NULL or start_date <= ?', end_date) + .where('due_date is NULL or due_date >= ?', start_date) + end + + strip_attributes :title + + alias_attribute :name, :title + end + + class_methods do + # Searches for timeboxes with a matching title or description. + # + # This method uses ILIKE on PostgreSQL + # + # query - The search query as a String + # + # Returns an ActiveRecord::Relation. + def search(query) + fuzzy_search(query, [:title, :description]) + end + + # Searches for timeboxes with a matching title. + # + # This method uses ILIKE on PostgreSQL + # + # query - The search query as a String + # + # Returns an ActiveRecord::Relation. + def search_title(query) + fuzzy_search(query, [:title]) + end + + def filter_by_state(timeboxes, state) + case state + when 'closed' then timeboxes.closed + when 'all' then timeboxes + else timeboxes.active + end + end + + def count_by_state + reorder(nil).group(:state).count + end + + def predefined_id?(id) + [Any.id, None.id, Upcoming.id, Started.id].include?(id) + end + + def predefined?(timebox) + predefined_id?(timebox&.id) + end + end + + def title=(value) + write_attribute(:title, sanitize_title(value)) if value.present? + end + + def timebox_name + model_name.singular + end + + def group_timebox? + group_id.present? + end + + def project_timebox? + project_id.present? + end + + def safe_title + title.to_slug.normalize.to_s + end + + def resource_parent + group || project + end + + def to_ability_name + model_name.singular + end + + def merge_requests_enabled? + if group_timebox? + # Assume that groups have at least one project with merge requests enabled. + # Otherwise, we would need to load all of the projects from the database. + true + elsif project_timebox? + project&.merge_requests_enabled? + end + end + + private + + # Timebox titles must be unique across project and group timeboxes + def uniqueness_of_title + if project + relation = self.class.for_projects_and_groups([project_id], [project.group&.id]) + elsif group + relation = self.class.for_projects_and_groups(group.projects.select(:id), [group.id]) + end + + title_exists = relation.find_by_title(title) + errors.add(:title, _("already being used for another group or project %{timebox_name}.") % { timebox_name: timebox_name }) if title_exists + end + + # Timebox should be either a project timebox or a group timebox + def timebox_type_check + if group_id && project_id + field = project_id_changed? ? :project_id : :group_id + errors.add(field, _("%{timebox_name} should belong either to a project or a group.") % { timebox_name: timebox_name }) + end + end + + def start_date_should_be_less_than_due_date + if due_date <= start_date + errors.add(:due_date, _("must be greater than start date")) + end + end + + def dates_within_4_digits + if start_date && start_date > Date.new(9999, 12, 31) + errors.add(:start_date, _("date must not be after 9999-12-31")) + end + + if due_date && due_date > Date.new(9999, 12, 31) + errors.add(:due_date, _("date must not be after 9999-12-31")) + end + end + + def sanitize_title(value) + CGI.unescape_html(Sanitize.clean(value.to_s)) + end +end diff --git a/app/models/concerns/update_project_statistics.rb b/app/models/concerns/update_project_statistics.rb index a84fb1cf56d..6cf012680d8 100644 --- a/app/models/concerns/update_project_statistics.rb +++ b/app/models/concerns/update_project_statistics.rb @@ -68,21 +68,11 @@ module UpdateProjectStatistics def schedule_update_project_statistic(delta) return if delta.zero? + return if project.nil? - if Feature.enabled?(:update_project_statistics_after_commit, default_enabled: true) - # Update ProjectStatistics after the transaction - run_after_commit do - ProjectStatistics.increment_statistic( - project_id, self.class.project_statistics_name, delta) - end - else - # Use legacy-way to update within transaction + run_after_commit do ProjectStatistics.increment_statistic( project_id, self.class.project_statistics_name, delta) - end - - run_after_commit do - next if project.nil? Namespaces::ScheduleAggregationWorker.perform_async( project.namespace_id) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 3bff7cb06c1..455c672cea3 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -2,6 +2,7 @@ class ContainerRepository < ApplicationRecord include Gitlab::Utils::StrongMemoize + include Gitlab::SQL::Pattern belongs_to :project @@ -17,6 +18,7 @@ class ContainerRepository < ApplicationRecord scope :for_group_and_its_subgroups, ->(group) do where(project_id: Project.for_group_and_its_subgroups(group).with_container_registry.select(:id)) end + scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } def self.exists_by_path?(path) where( diff --git a/app/models/cycle_analytics/group_level.rb b/app/models/cycle_analytics/group_level.rb deleted file mode 100644 index a41e1375484..00000000000 --- a/app/models/cycle_analytics/group_level.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module CycleAnalytics - class GroupLevel - include LevelBase - attr_reader :options, :group - - def initialize(group:, options:) - @group = group - @options = options.merge(group: group) - end - - def summary - @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group, options: options).data - end - - def permissions(*) - STAGES.each_with_object({}) do |stage, obj| - obj[stage] = true - end - end - - def stats - @stats ||= STAGES.map do |stage_name| - self[stage_name].as_json(serializer: GroupAnalyticsStageSerializer) - end - end - end -end diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 69245710f01..395260b5201 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -7,7 +7,8 @@ class DeployToken < ApplicationRecord include Gitlab::Utils::StrongMemoize add_authentication_token_field :token, encrypted: :optional - AVAILABLE_SCOPES = %i(read_repository read_registry write_registry).freeze + AVAILABLE_SCOPES = %i(read_repository read_registry write_registry + read_package_registry write_package_registry).freeze GITLAB_DEPLOY_TOKEN_NAME = 'gitlab-deploy-token' default_value_for(:expires_at) { Forever.date } @@ -105,7 +106,7 @@ class DeployToken < ApplicationRecord end def ensure_at_least_one_scope - errors.add(:base, _("Scopes can't be blank")) unless read_repository || read_registry || write_registry + errors.add(:base, _("Scopes can't be blank")) unless scopes.any? end def default_username diff --git a/app/models/design_management.rb b/app/models/design_management.rb new file mode 100644 index 00000000000..81e170f7e59 --- /dev/null +++ b/app/models/design_management.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module DesignManagement + DESIGN_IMAGE_SIZES = %w(v432x230).freeze + + def self.designs_directory + 'designs' + end + + def self.table_name_prefix + 'design_management_' + end +end diff --git a/app/models/design_management/action.rb b/app/models/design_management/action.rb new file mode 100644 index 00000000000..ecd7973a523 --- /dev/null +++ b/app/models/design_management/action.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_dependency 'design_management' + +module DesignManagement + class Action < ApplicationRecord + include WithUploads + + self.table_name = "#{DesignManagement.table_name_prefix}designs_versions" + + mount_uploader :image_v432x230, DesignManagement::DesignV432x230Uploader + + belongs_to :design, class_name: "DesignManagement::Design", inverse_of: :actions + belongs_to :version, class_name: "DesignManagement::Version", inverse_of: :actions + + enum event: { creation: 0, modification: 1, deletion: 2 } + + # we assume sequential ordering. + scope :ordered, -> { order(version_id: :asc) } + + # For each design, only select the most recent action + scope :most_recent, -> do + selection = Arel.sql("DISTINCT ON (#{table_name}.design_id) #{table_name}.*") + + order(arel_table[:design_id].asc, arel_table[:version_id].desc).select(selection) + end + + # Find all records created before or at the given version, or all if nil + scope :up_to_version, ->(version = nil) do + case version + when nil + all + when DesignManagement::Version + where(arel_table[:version_id].lteq(version.id)) + when ::Gitlab::Git::COMMIT_ID + versions = DesignManagement::Version.arel_table + subquery = versions.project(versions[:id]).where(versions[:sha].eq(version)) + where(arel_table[:version_id].lteq(subquery)) + else + raise ArgumentError, "Expected a DesignManagement::Version, got #{version}" + end + end + end +end diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb new file mode 100644 index 00000000000..e9b69eab7a7 --- /dev/null +++ b/app/models/design_management/design.rb @@ -0,0 +1,266 @@ +# frozen_string_literal: true + +module DesignManagement + class Design < ApplicationRecord + include Importable + include Noteable + include Gitlab::FileTypeDetection + include Gitlab::Utils::StrongMemoize + include Referable + include Mentionable + include WhereComposite + + belongs_to :project, inverse_of: :designs + belongs_to :issue + + has_many :actions + has_many :versions, through: :actions, class_name: 'DesignManagement::Version', inverse_of: :designs + # This is a polymorphic association, so we can't count on FK's to delete the + # data + has_many :notes, as: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_many :user_mentions, class_name: 'DesignUserMention', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + validates :project, :filename, presence: true + validates :issue, presence: true, unless: :importing? + validates :filename, uniqueness: { scope: :issue_id } + validate :validate_file_is_image + + alias_attribute :title, :filename + + # Pre-fetching scope to include the data necessary to construct a + # reference using `to_reference`. + scope :for_reference, -> { includes(issue: [{ project: [:route, :namespace] }]) } + + # A design can be uniquely identified by issue_id and filename + # Takes one or more sets of composite IDs of the form: + # `{issue_id: Integer, filename: String}`. + # + # @see WhereComposite::where_composite + # + # e.g: + # + # by_issue_id_and_filename(issue_id: 1, filename: 'homescreen.jpg') + # by_issue_id_and_filename([]) # returns ActiveRecord::NullRelation + # by_issue_id_and_filename([ + # { issue_id: 1, filename: 'homescreen.jpg' }, + # { issue_id: 2, filename: 'homescreen.jpg' }, + # { issue_id: 1, filename: 'menu.png' } + # ]) + # + scope :by_issue_id_and_filename, ->(composites) do + where_composite(%i[issue_id filename], composites) + end + + # Find designs visible at the given version + # + # @param version [nil, DesignManagement::Version]: + # the version at which the designs must be visible + # Passing `nil` is the same as passing the most current version + # + # Restricts to designs + # - created at least *before* the given version + # - not deleted as of the given version. + # + # As a query, we ascertain this by finding the last event prior to + # (or equal to) the cut-off, and seeing whether that version was a deletion. + scope :visible_at_version, -> (version) do + deletion = ::DesignManagement::Action.events[:deletion] + designs = arel_table + actions = ::DesignManagement::Action + .most_recent.up_to_version(version) + .arel.as('most_recent_actions') + + join = designs.join(actions) + .on(actions[:design_id].eq(designs[:id])) + + joins(join.join_sources).where(actions[:event].not_eq(deletion)).order(:id) + end + + scope :with_filename, -> (filenames) { where(filename: filenames) } + scope :on_issue, ->(issue) { where(issue_id: issue) } + + # Scope called by our REST API to avoid N+1 problems + scope :with_api_entity_associations, -> { preload(:issue) } + + # A design is current if the most recent event is not a deletion + scope :current, -> { visible_at_version(nil) } + + def status + if new_design? + :new + elsif deleted? + :deleted + else + :current + end + end + + def deleted? + most_recent_action&.deletion? + end + + # A design is visible_in? a version if: + # * it was created before that version + # * the most recent action before the version was not a deletion + def visible_in?(version) + map = strong_memoize(:visible_in) do + Hash.new do |h, k| + h[k] = self.class.visible_at_version(k).where(id: id).exists? + end + end + + map[version] + end + + def most_recent_action + strong_memoize(:most_recent_action) { actions.ordered.last } + end + + # A reference for a design is the issue reference, indexed by the filename + # with an optional infix when full. + # + # e.g. + # #123[homescreen.png] + # other-project#72[sidebar.jpg] + # #38/designs[transition.gif] + # #12["filename with [] in it.jpg"] + def to_reference(from = nil, full: false) + infix = full ? '/designs' : '' + totally_simple = %r{ \A #{self.class.simple_file_name} \z }x + safe_name = if totally_simple.match?(filename) + filename + elsif filename =~ /[<>]/ + %Q{base64:#{Base64.strict_encode64(filename)}} + else + escaped = filename.gsub(%r{[\\"]}) { |x| "\\#{x}" } + %Q{"#{escaped}"} + end + + "#{issue.to_reference(from, full: full)}#{infix}[#{safe_name}]" + end + + def self.reference_pattern + @reference_pattern ||= begin + # Filenames can be escaped with double quotes to name filenames + # that include square brackets, or other special characters + %r{ + #{Issue.reference_pattern} + (\/designs)? + \[ + (?<design> #{simple_file_name} | #{quoted_file_name} | #{base_64_encoded_name}) + \] + }x + end + end + + def self.simple_file_name + %r{ + (?<simple_file_name> + ( \w | [_:,'-] | \. | \s )+ + \. + \w+ + ) + }x + end + + def self.base_64_encoded_name + %r{ + base64: + (?<base_64_encoded_name> + [A-Za-z0-9+\n]+ + =? + ) + }x + end + + def self.quoted_file_name + %r{ + " + (?<escaped_filename> + (\\ \\ | \\ " | [^"\\])+ + ) + " + }x + end + + def self.link_reference_pattern + @link_reference_pattern ||= begin + exts = SAFE_IMAGE_EXT + DANGEROUS_IMAGE_EXT + path_segment = %r{issues/#{Gitlab::Regex.issue}/designs} + filename_pattern = %r{(?<simple_file_name>[a-z0-9_=-]+\.(#{exts.join('|')}))}i + + super(path_segment, filename_pattern) + end + end + + def to_ability_name + 'design' + end + + def description + '' + end + + def new_design? + strong_memoize(:new_design) { actions.none? } + end + + def full_path + @full_path ||= File.join(DesignManagement.designs_directory, "issue-#{issue.iid}", filename) + end + + def diff_refs + strong_memoize(:diff_refs) { head_version&.diff_refs } + end + + def clear_version_cache + [versions, actions].each(&:reset) + %i[new_design diff_refs head_sha visible_in most_recent_action].each do |key| + clear_memoization(key) + end + end + + def repository + project.design_repository + end + + def user_notes_count + user_notes_count_service.count + end + + def after_note_changed(note) + user_notes_count_service.delete_cache unless note.system? + end + alias_method :after_note_created, :after_note_changed + alias_method :after_note_destroyed, :after_note_changed + + private + + def head_version + strong_memoize(:head_sha) { versions.ordered.first } + end + + def allow_dangerous_images? + Feature.enabled?(:design_management_allow_dangerous_images, project) + end + + def valid_file_extensions + allow_dangerous_images? ? (SAFE_IMAGE_EXT + DANGEROUS_IMAGE_EXT) : SAFE_IMAGE_EXT + end + + def validate_file_is_image + unless image? || (dangerous_image? && allow_dangerous_images?) + message = _('does not have a supported extension. Only %{extension_list} are supported') % { + extension_list: valid_file_extensions.to_sentence + } + errors.add(:filename, message) + end + end + + def user_notes_count_service + strong_memoize(:user_notes_count_service) do + ::DesignManagement::DesignUserNotesCountService.new(self) # rubocop: disable CodeReuse/ServiceClass + end + end + end +end diff --git a/app/models/design_management/design_action.rb b/app/models/design_management/design_action.rb new file mode 100644 index 00000000000..22baa916296 --- /dev/null +++ b/app/models/design_management/design_action.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module DesignManagement + # Parameter object which is a tuple of the database record and the + # last gitaly call made to change it. This serves to perform the + # logical mapping from git action to database representation. + class DesignAction + include ActiveModel::Validations + + EVENT_FOR_GITALY_ACTION = { + create: DesignManagement::Action.events[:creation], + update: DesignManagement::Action.events[:modification], + delete: DesignManagement::Action.events[:deletion] + }.freeze + + attr_reader :design, :action, :content + + delegate :issue_id, to: :design + + validates :design, presence: true + validates :action, presence: true, inclusion: { in: EVENT_FOR_GITALY_ACTION.keys } + validates :content, + absence: { if: :forbids_content?, + message: 'this action forbids content' }, + presence: { if: :needs_content?, + message: 'this action needs content' } + + # Parameters: + # - design [DesignManagement::Design]: the design that was changed + # - action [Symbol]: the action that gitaly performed + def initialize(design, action, content = nil) + @design, @action, @content = design, action, content + validate! + end + + def row_attrs(version) + { design_id: design.id, version_id: version.id, event: event } + end + + def gitaly_action + { action: action, file_path: design.full_path, content: content }.compact + end + + # This action has been performed - do any post-creation actions + # such as clearing method caches. + def performed + design.clear_version_cache + end + + private + + def needs_content? + action != :delete + end + + def forbids_content? + action == :delete + end + + def event + EVENT_FOR_GITALY_ACTION[action] + end + end +end diff --git a/app/models/design_management/design_at_version.rb b/app/models/design_management/design_at_version.rb new file mode 100644 index 00000000000..b4cafb93c2c --- /dev/null +++ b/app/models/design_management/design_at_version.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +# Tuple of design and version +# * has a composite ID, with lazy_find +module DesignManagement + class DesignAtVersion + include ActiveModel::Validations + include GlobalID::Identification + include Gitlab::Utils::StrongMemoize + + attr_reader :version + attr_reader :design + + validates :version, presence: true + validates :design, presence: true + + validate :design_and_version_belong_to_the_same_issue + validate :design_and_version_have_issue_id + + def initialize(design: nil, version: nil) + @design, @version = design, version + end + + def self.instantiate(attrs) + new(attrs).tap { |obj| obj.validate! } + end + + # The ID, needed by GraphQL types and as part of the Lazy-fetch + # protocol, includes information about both the design and the version. + # + # The particular format is not interesting, and should be treated as opaque + # by all callers. + def id + "#{design.id}.#{version.id}" + end + + def ==(other) + return false unless other && self.class == other.class + + other.id == id + end + + alias_method :eql?, :== + + def self.lazy_find(id) + BatchLoader.for(id).batch do |ids, callback| + find(ids).each do |record| + callback.call(record.id, record) + end + end + end + + def self.find(ids) + pairs = ids.map { |id| id.split('.').map(&:to_i) } + + design_ids = pairs.map(&:first).uniq + version_ids = pairs.map(&:second).uniq + + designs = ::DesignManagement::Design + .where(id: design_ids) + .index_by(&:id) + + versions = ::DesignManagement::Version + .where(id: version_ids) + .index_by(&:id) + + pairs.map do |(design_id, version_id)| + design = designs[design_id] + version = versions[version_id] + + obj = new(design: design, version: version) + + obj if obj.valid? + end.compact + end + + def status + if not_created_yet? + :not_created_yet + elsif deleted? + :deleted + else + :current + end + end + + def deleted? + action&.deletion? + end + + def not_created_yet? + action.nil? + end + + private + + def action + strong_memoize(:most_recent_action) do + ::DesignManagement::Action + .most_recent.up_to_version(version) + .find_by(design: design) + end + end + + def design_and_version_belong_to_the_same_issue + id_a, id_b = [design, version].map { |obj| obj&.issue_id } + + return if id_a == id_b + + errors.add(:issue, 'must be the same on design and version') + end + + def design_and_version_have_issue_id + return if [design, version].all? { |obj| obj.try(:issue_id).present? } + + errors.add(:issue, 'must be present on both design and version') + end + end +end diff --git a/app/models/design_management/design_collection.rb b/app/models/design_management/design_collection.rb new file mode 100644 index 00000000000..18d1541e9c7 --- /dev/null +++ b/app/models/design_management/design_collection.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module DesignManagement + class DesignCollection + attr_reader :issue + + delegate :designs, :project, to: :issue + + def initialize(issue) + @issue = issue + end + + def find_or_create_design!(filename:) + designs.find { |design| design.filename == filename } || + designs.safe_find_or_create_by!(project: project, filename: filename) + end + + def versions + @versions ||= DesignManagement::Version.for_designs(designs) + end + + def repository + project.design_repository + end + + def designs_by_filename(filenames) + designs.current.where(filename: filenames) + end + end +end diff --git a/app/models/design_management/repository.rb b/app/models/design_management/repository.rb new file mode 100644 index 00000000000..985d6317d5d --- /dev/null +++ b/app/models/design_management/repository.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module DesignManagement + class Repository < ::Repository + extend ::Gitlab::Utils::Override + + # We define static git attributes for the design repository as this + # repository is entirely GitLab-managed rather than user-facing. + # + # Enable all uploaded files to be stored in LFS. + MANAGED_GIT_ATTRIBUTES = <<~GA.freeze + /#{DesignManagement.designs_directory}/* filter=lfs diff=lfs merge=lfs -text + GA + + def initialize(project) + full_path = project.full_path + Gitlab::GlRepository::DESIGN.path_suffix + disk_path = project.disk_path + Gitlab::GlRepository::DESIGN.path_suffix + + super(full_path, project, shard: project.repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::DESIGN) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def info_attributes + @info_attributes ||= Gitlab::Git::AttributesParser.new(MANAGED_GIT_ATTRIBUTES) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def attributes(path) + info_attributes.attributes(path) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def gitattribute(path, name) + attributes(path)[name] + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def attributes_at(_ref = nil) + info_attributes + end + + override :copy_gitattributes + def copy_gitattributes(_ref = nil) + true + end + end +end diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb new file mode 100644 index 00000000000..6be98fe3d44 --- /dev/null +++ b/app/models/design_management/version.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +module DesignManagement + class Version < ApplicationRecord + include Importable + include ShaAttribute + include AfterCommitQueue + include Gitlab::Utils::StrongMemoize + extend Gitlab::ExclusiveLeaseHelpers + + NotSameIssue = Class.new(StandardError) + + class CouldNotCreateVersion < StandardError + attr_reader :sha, :issue_id, :actions + + def initialize(sha, issue_id, actions) + @sha, @issue_id, @actions = sha, issue_id, actions + end + + def message + "could not create version from commit: #{sha}" + end + + def sentry_extra_data + { + sha: sha, + issue_id: issue_id, + design_ids: actions.map { |a| a.design.id } + } + end + end + + belongs_to :issue + belongs_to :author, class_name: 'User' + has_many :actions + has_many :designs, + through: :actions, + class_name: "DesignManagement::Design", + source: :design, + inverse_of: :versions + + validates :designs, presence: true, unless: :importing? + validates :sha, presence: true + validates :sha, uniqueness: { case_sensitive: false, scope: :issue_id } + validates :author, presence: true + # We are not validating the issue object as it incurs an extra query to fetch + # the record from the DB. Instead, we rely on the foreign key constraint to + # ensure referential integrity. + validates :issue_id, presence: true, unless: :importing? + + sha_attribute :sha + + delegate :project, to: :issue + + scope :for_designs, -> (designs) do + where(id: ::DesignManagement::Action.where(design_id: designs).select(:version_id)).distinct + end + scope :earlier_or_equal_to, -> (version) { where("(#{table_name}.id) <= ?", version) } # rubocop:disable GitlabSecurity/SqlInjection + scope :ordered, -> { order(id: :desc) } + scope :for_issue, -> (issue) { where(issue: issue) } + scope :by_sha, -> (sha) { where(sha: sha) } + + # This is the one true way to create a Version. + # + # This method means you can avoid the paradox of versions being invalid without + # designs, and not being able to add designs without a saved version. Also this + # method inserts designs in bulk, rather than one by one. + # + # Before calling this method, callers must guard against concurrent + # modification by obtaining the lock on the design repository. See: + # `DesignManagement::Version.with_lock`. + # + # Parameters: + # - design_actions [DesignManagement::DesignAction]: + # the actions that have been performed in the repository. + # - sha [String]: + # the SHA of the commit that performed them + # - author [User]: + # the user who performed the commit + # returns [DesignManagement::Version] + def self.create_for_designs(design_actions, sha, author) + issue_id, not_uniq = design_actions.map(&:issue_id).compact.uniq + raise NotSameIssue, 'All designs must belong to the same issue!' if not_uniq + + transaction do + version = new(sha: sha, issue_id: issue_id, author: author) + version.save(validate: false) # We need it to have an ID. Validate later when designs are present + + rows = design_actions.map { |action| action.row_attrs(version) } + + Gitlab::Database.bulk_insert(::DesignManagement::Action.table_name, rows) + version.designs.reset + version.validate! + design_actions.each(&:performed) + + version + end + rescue + raise CouldNotCreateVersion.new(sha, issue_id, design_actions) + end + + CREATION_TTL = 5.seconds + RETRY_DELAY = ->(num) { 0.2.seconds * num**2 } + + def self.with_lock(project_id, repository, &block) + key = "with_lock:#{name}:{#{project_id}}" + + in_lock(key, ttl: CREATION_TTL, retries: 5, sleep_sec: RETRY_DELAY) do |_retried| + repository.create_if_not_exists + yield + end + end + + def designs_by_event + actions + .includes(:design) + .group_by(&:event) + .transform_values { |group| group.map(&:design) } + end + + def author + super || (commit_author if persisted?) + end + + def diff_refs + strong_memoize(:diff_refs) { commit&.diff_refs } + end + + def reset + %i[diff_refs commit].each { |k| clear_memoization(k) } + super + end + + private + + def commit_author + commit&.author + end + + def commit + strong_memoize(:commit) { issue.project.design_repository.commit(sha) } + end + end +end diff --git a/app/models/design_user_mention.rb b/app/models/design_user_mention.rb new file mode 100644 index 00000000000..baf4db29a0f --- /dev/null +++ b/app/models/design_user_mention.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class DesignUserMention < UserMention + belongs_to :design, class_name: 'DesignManagement::Design' + belongs_to :note +end diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index e3df61dadae..ff39dbb59f3 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -9,7 +9,7 @@ class DiffNote < Note include Gitlab::Utils::StrongMemoize def self.noteable_types - %w(MergeRequest Commit) + %w(MergeRequest Commit DesignManagement::Design) end validates :original_position, presence: true @@ -60,6 +60,8 @@ class DiffNote < Note # Returns the diff file from `position` def latest_diff_file strong_memoize(:latest_diff_file) do + next if for_design? + position.diff_file(repository) end end @@ -67,6 +69,8 @@ class DiffNote < Note # Returns the diff file from `original_position` def diff_file strong_memoize(:diff_file) do + next if for_design? + enqueue_diff_file_creation_job if should_create_diff_file? fetch_diff_file @@ -145,7 +149,7 @@ class DiffNote < Note end def supported? - for_commit? || self.noteable.has_complete_diff_refs? + for_commit? || for_design? || self.noteable.has_complete_diff_refs? end def set_line_code @@ -184,5 +188,3 @@ class DiffNote < Note noteable.respond_to?(:repository) ? noteable.repository : project.repository end end - -DiffNote.prepend_if_ee('::EE::DiffNote') diff --git a/app/models/email.rb b/app/models/email.rb index 580633d3232..c5154267ff0 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -6,7 +6,8 @@ class Email < ApplicationRecord belongs_to :user, optional: false - validates :email, presence: true, uniqueness: true, devise_email: true + validates :email, presence: true, uniqueness: true + validate :validate_email_format validate :unique_email, if: ->(email) { email.email_changed? } scope :confirmed, -> { where.not(confirmed_at: nil) } @@ -14,9 +15,14 @@ class Email < ApplicationRecord after_commit :update_invalid_gpg_signatures, if: -> { previous_changes.key?('confirmed_at') } devise :confirmable + + # This module adds async behaviour to Devise emails + # and should be added after Devise modules are initialized. + include AsyncDeviseEmail + self.reconfirmable = false # currently email can't be changed, no need to reconfirm - delegate :username, to: :user + delegate :username, :can?, to: :user def email=(value) write_attribute(:email, value.downcase.strip) @@ -30,6 +36,10 @@ class Email < ApplicationRecord user.accept_pending_invitations! end + def validate_email_format + self.errors.add(:email, I18n.t(:invalid, scope: 'valid_email.validations.email')) unless ValidateEmail.valid?(self.email) + end + # once email is confirmed, update the gpg signatures def update_invalid_gpg_signatures user.update_invalid_gpg_signatures if confirmed? diff --git a/app/models/environment.rb b/app/models/environment.rb index b2391f33aca..21044771bbb 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -8,6 +8,7 @@ class Environment < ApplicationRecord self.reactive_cache_refresh_interval = 1.minute self.reactive_cache_lifetime = 55.seconds self.reactive_cache_hard_limit = 10.megabytes + self.reactive_cache_work_type = :external_dependency belongs_to :project, required: true @@ -151,6 +152,14 @@ class Environment < ApplicationRecord .preload(:user, :metadata, :deployment) end + def count_by_state + environments_count_by_state = group(:state).count + + valid_states.each_with_object({}) do |state, count_hash| + count_hash[state] = environments_count_by_state[state.to_s] || 0 + end + end + private def cte_for_deployments_with_stop_action diff --git a/app/models/epic.rb b/app/models/epic.rb index 04e19c17e18..e09dc1080e6 100644 --- a/app/models/epic.rb +++ b/app/models/epic.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # Placeholder class for model that is implemented in EE -# It reserves '&' as a reference prefix, but the table does not exists in CE +# It reserves '&' as a reference prefix, but the table does not exist in FOSS class Epic < ApplicationRecord include IgnorableColumns diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index 133850b6ab6..fa32c8a5450 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -22,6 +22,7 @@ module ErrorTracking }x.freeze self.reactive_cache_key = ->(setting) { [setting.class.model_name.singular, setting.project_id] } + self.reactive_cache_work_type = :external_dependency belongs_to :project diff --git a/app/models/event.rb b/app/models/event.rb index 447ab753421..12b85697690 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -96,6 +96,8 @@ class Event < ApplicationRecord end scope :for_milestone_id, ->(milestone_id) { where(target_type: "Milestone", target_id: milestone_id) } + scope :for_wiki_meta, ->(meta) { where(target_type: 'WikiPage::Meta', target_id: meta.id) } + scope :created_at, ->(time) { where(created_at: time) } # Authors are required as they're used to display who pushed data. # @@ -313,6 +315,10 @@ class Event < ApplicationRecord note? && target && target.for_personal_snippet? end + def design_note? + note? && note.for_design? + end + def note_target target.noteable end @@ -380,6 +386,11 @@ class Event < ApplicationRecord protected + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + # + # TODO Refactor this method so we no longer need to disable the above cops + # https://gitlab.com/gitlab-org/gitlab/-/issues/216879. def capability @capability ||= begin if push_action? || commit_note? @@ -396,9 +407,13 @@ class Event < ApplicationRecord :read_milestone elsif wiki_page? :read_wiki + elsif design_note? + :read_design end end end + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity private diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb index d0cec0e9fc6..43de7454cb7 100644 --- a/app/models/global_milestone.rb +++ b/app/models/global_milestone.rb @@ -2,7 +2,6 @@ # Global Milestones are milestones that can be shared across multiple projects class GlobalMilestone include Milestoneish - include_if_ee('::EE::GlobalMilestone') # rubocop: disable Cop/InjectEnterpriseEditionModule STATE_COUNT_HASH = { opened: 0, closed: 0, all: 0 }.freeze @@ -11,7 +10,7 @@ class GlobalMilestone delegate :title, :state, :due_date, :start_date, :participants, :project, :group, :expires_at, :closed?, :iid, :group_milestone?, :safe_title, - :milestoneish_id, :resource_parent, :releases, to: :milestone + :timebox_id, :milestoneish_id, :resource_parent, :releases, to: :milestone def to_hash { @@ -105,3 +104,5 @@ class GlobalMilestone true end end + +GlobalMilestone.include_if_ee('::EE::GlobalMilestone') diff --git a/app/models/group.rb b/app/models/group.rb index 55a2c4ba9a9..04cb6b8b4da 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -30,6 +30,7 @@ class Group < Namespace has_many :members_and_requesters, as: :source, class_name: 'GroupMember' has_many :milestones + has_many :iterations has_many :shared_group_links, foreign_key: :shared_with_group_id, class_name: 'GroupGroupLink' has_many :shared_with_group_links, foreign_key: :shared_group_id, class_name: 'GroupGroupLink' has_many :shared_groups, through: :shared_group_links, source: :shared_group @@ -59,6 +60,8 @@ class Group < Namespace has_many :import_failures, inverse_of: :group + has_one :import_state, class_name: 'GroupImportState', inverse_of: :group + has_many :group_deploy_tokens has_many :deploy_tokens, through: :group_deploy_tokens @@ -168,7 +171,7 @@ class Group < Namespace notification_settings.find { |n| n.notification_email.present? }&.notification_email end - def to_reference(_from = nil, full: nil) + def to_reference(_from = nil, target_project: nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end @@ -302,9 +305,10 @@ class Group < Namespace # rubocop: enable CodeReuse/ServiceClass # rubocop: disable CodeReuse/ServiceClass - def refresh_members_authorized_projects(blocking: true) - UserProjectAccessChangedService.new(user_ids_for_project_authorizations) - .execute(blocking: blocking) + def refresh_members_authorized_projects(blocking: true, priority: UserProjectAccessChangedService::HIGH_PRIORITY) + UserProjectAccessChangedService + .new(user_ids_for_project_authorizations) + .execute(blocking: blocking, priority: priority) end # rubocop: enable CodeReuse/ServiceClass @@ -332,6 +336,11 @@ class Group < Namespace .where(source_id: source_ids) end + def members_from_self_and_ancestors_with_effective_access_level + members_with_parents.select([:user_id, 'MAX(access_level) AS access_level']) + .group(:user_id) + end + def members_with_descendants GroupMember .active_without_invites_and_requests @@ -475,14 +484,14 @@ class Group < Namespace false end - def wiki_access_level - # TODO: Remove this method once we implement group-level features. - # https://gitlab.com/gitlab-org/gitlab/-/issues/208412 - if Feature.enabled?(:group_wiki, self) - ProjectFeature::ENABLED - else - ProjectFeature::DISABLED - end + def execute_hooks(data, hooks_scope) + # NOOP + # TODO: group hooks https://gitlab.com/gitlab-org/gitlab/-/issues/216904 + end + + def execute_services(data, hooks_scope) + # NOOP + # TODO: group hooks https://gitlab.com/gitlab-org/gitlab/-/issues/216904 end private @@ -516,8 +525,6 @@ class Group < Namespace end def max_member_access_for_user_from_shared_groups(user) - return unless Feature.enabled?(:share_group_with_group, default_enabled: true) - group_group_link_table = GroupGroupLink.arel_table group_member_table = GroupMember.arel_table diff --git a/app/models/group_import_state.rb b/app/models/group_import_state.rb new file mode 100644 index 00000000000..7773b887249 --- /dev/null +++ b/app/models/group_import_state.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class GroupImportState < ApplicationRecord + self.primary_key = :group_id + + belongs_to :group, inverse_of: :import_state + + validates :group, :status, :jid, presence: true + + state_machine :status, initial: :created do + state :created, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :failed, value: -1 + + event :start do + transition created: :started + end + + event :finish do + transition started: :finished + end + + event :fail_op do + transition any => :failed + end + + after_transition any => :failed do |state, transition| + last_error = transition.args.first + + state.update_column(:last_error, last_error) if last_error + end + end +end diff --git a/app/models/group_milestone.rb b/app/models/group_milestone.rb index 87338512d99..60e97174e50 100644 --- a/app/models/group_milestone.rb +++ b/app/models/group_milestone.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true # Group Milestones are milestones that can be shared among many projects within the same group class GroupMilestone < GlobalMilestone - include_if_ee('::EE::GroupMilestone') # rubocop: disable Cop/InjectEnterpriseEditionModule attr_reader :group, :milestones def self.build_collection(group, projects, params) @@ -46,3 +45,5 @@ class GroupMilestone < GlobalMilestone true end end + +GroupMilestone.include_if_ee('::EE::GroupMilestone') diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index bc480b14e67..71494b6de4d 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -3,6 +3,9 @@ class ProjectHook < WebHook include TriggerableHooks include Presentable + include Limitable + + self.limit_scope = :project triggerable_hooks [ :push_hooks, diff --git a/app/models/internal_id_enums.rb b/app/models/internal_id_enums.rb index 2f7d7aeff2f..125ae7573b6 100644 --- a/app/models/internal_id_enums.rb +++ b/app/models/internal_id_enums.rb @@ -3,7 +3,18 @@ module InternalIdEnums def self.usage_resources # when adding new resource, make sure it doesn't conflict with EE usage_resources - { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5, operations_feature_flags: 6 } + { + issues: 0, + merge_requests: 1, + deployments: 2, + milestones: 3, + epics: 4, + ci_pipelines: 5, + operations_feature_flags: 6, + operations_user_lists: 7, + alert_management_alerts: 8, + sprints: 9 # iterations + } end end diff --git a/app/models/issue.rb b/app/models/issue.rb index cdd7429bc58..a04ac412940 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -17,6 +17,7 @@ class Issue < ApplicationRecord include IgnorableColumns include MilestoneEventable include WhereComposite + include StateEventable DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze @@ -29,9 +30,12 @@ class Issue < ApplicationRecord SORTING_PREFERENCE_FIELD = :issues_sort belongs_to :project - belongs_to :moved_to, class_name: 'Issue' belongs_to :duplicated_to, class_name: 'Issue' belongs_to :closed_by, class_name: 'User' + belongs_to :iteration, foreign_key: 'sprint_id' + + belongs_to :moved_to, class_name: 'Issue' + has_one :moved_from, class_name: 'Issue', foreign_key: :moved_to_id has_internal_id :iid, scope: :project, track_if: -> { !importing? }, init: ->(s) { s&.project&.issues&.maximum(:iid) } @@ -46,8 +50,15 @@ class Issue < ApplicationRecord has_many :zoom_meetings has_many :user_mentions, class_name: "IssueUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :sent_notifications, as: :noteable + has_many :designs, class_name: 'DesignManagement::Design', inverse_of: :issue + has_many :design_versions, class_name: 'DesignManagement::Version', inverse_of: :issue do + def most_recent + ordered.first + end + end has_one :sentry_issue + has_one :alert_management_alert, class_name: 'AlertManagement::Alert' accepts_nested_attributes_for :sentry_issue @@ -63,6 +74,7 @@ class Issue < ApplicationRecord scope :due_before, ->(date) { where('issues.due_date < ?', date) } scope :due_between, ->(from_date, to_date) { where('issues.due_date >= ?', from_date).where('issues.due_date <= ?', to_date) } scope :due_tomorrow, -> { where(due_date: Date.tomorrow) } + scope :not_authored_by, ->(user) { where.not(author_id: user) } scope :order_due_date_asc, -> { reorder(::Gitlab::Database.nulls_last_order('due_date', 'ASC')) } scope :order_due_date_desc, -> { reorder(::Gitlab::Database.nulls_last_order('due_date', 'DESC')) } @@ -73,11 +85,13 @@ class Issue < ApplicationRecord scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } scope :with_api_entity_associations, -> { preload(:timelogs, :assignees, :author, :notes, :labels, project: [:route, { namespace: :route }] ) } + scope :with_label_attributes, ->(label_attributes) { joins(:labels).where(labels: label_attributes) } scope :public_only, -> { where(confidential: false) } scope :confidential_only, -> { where(confidential: true) } scope :counts_by_state, -> { reorder(nil).group(:state_id).count } + scope :with_alert_management_alerts, -> { joins(:alert_management_alert) } # An issue can be uniquely identified by project_id and iid # Takes one or more sets of composite IDs, expressed as hash-like records of @@ -330,6 +344,10 @@ class Issue < ApplicationRecord previous_changes['updated_at']&.first || updated_at end + def design_collection + @design_collection ||= ::DesignManagement::DesignCollection.new(self) + end + private def ensure_metrics @@ -343,7 +361,7 @@ class Issue < ApplicationRecord # for performance reasons, check commit: 002ad215818450d2cbbc5fa065850a953dc7ada8 # Make sure to sync this method with issue_policy.rb def readable_by?(user) - if user.admin? + if user.can_read_all_resources? true elsif project.owner == user true diff --git a/app/models/iteration.rb b/app/models/iteration.rb new file mode 100644 index 00000000000..1acd08f2063 --- /dev/null +++ b/app/models/iteration.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +class Iteration < ApplicationRecord + include Timebox + + self.table_name = 'sprints' + + attr_accessor :skip_future_date_validation + + STATE_ENUM_MAP = { + upcoming: 1, + started: 2, + closed: 3 + }.with_indifferent_access.freeze + + include AtomicInternalId + + has_many :issues, foreign_key: 'sprint_id' + has_many :merge_requests, foreign_key: 'sprint_id' + + belongs_to :project + belongs_to :group + + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.iterations&.maximum(:iid) } + has_internal_id :iid, scope: :group, init: ->(s) { s&.group&.iterations&.maximum(:iid) } + + validates :start_date, presence: true + validates :due_date, presence: true + + validate :dates_do_not_overlap, if: :start_or_due_dates_changed? + validate :future_date, if: :start_or_due_dates_changed?, unless: :skip_future_date_validation + + scope :upcoming, -> { with_state(:upcoming) } + scope :started, -> { with_state(:started) } + + state_machine :state_enum, initial: :upcoming do + event :start do + transition upcoming: :started + end + + event :close do + transition [:upcoming, :started] => :closed + end + + state :upcoming, value: Iteration::STATE_ENUM_MAP[:upcoming] + state :started, value: Iteration::STATE_ENUM_MAP[:started] + state :closed, value: Iteration::STATE_ENUM_MAP[:closed] + end + + # Alias to state machine .with_state_enum method + # This needs to be defined after the state machine block to avoid errors + class << self + alias_method :with_state, :with_state_enum + alias_method :with_states, :with_state_enums + + def filter_by_state(iterations, state) + case state + when 'closed' then iterations.closed + when 'started' then iterations.started + when 'opened' then iterations.started.or(iterations.upcoming) + when 'all' then iterations + else iterations.upcoming + end + end + end + + def state + STATE_ENUM_MAP.key(state_enum) + end + + def state=(value) + self.state_enum = STATE_ENUM_MAP[value] + end + + private + + def start_or_due_dates_changed? + start_date_changed? || due_date_changed? + end + + # ensure dates do not overlap with other Iterations in the same group/project + def dates_do_not_overlap + return unless resource_parent.iterations.within_timeframe(start_date, due_date).exists? + + errors.add(:base, s_("Iteration|Dates cannot overlap with other existing Iterations")) + end + + # ensure dates are in the future + def future_date + if start_date_changed? + errors.add(:start_date, s_("Iteration|cannot be in the past")) if start_date < Date.current + errors.add(:start_date, s_("Iteration|cannot be more than 500 years in the future")) if start_date > 500.years.from_now + end + + if due_date_changed? + errors.add(:due_date, s_("Iteration|cannot be in the past")) if due_date < Date.current + errors.add(:due_date, s_("Iteration|cannot be more than 500 years in the future")) if due_date > 500.years.from_now + end + end +end diff --git a/app/models/jira_import_state.rb b/app/models/jira_import_state.rb index bde2795e7b8..92147794e88 100644 --- a/app/models/jira_import_state.rb +++ b/app/models/jira_import_state.rb @@ -3,6 +3,7 @@ class JiraImportState < ApplicationRecord include AfterCommitQueue include ImportState::SidekiqJobTracker + include UsageStatistics self.table_name = 'jira_imports' @@ -46,7 +47,7 @@ class JiraImportState < ApplicationRecord after_transition initial: :scheduled do |state, _| state.run_after_commit do job_id = Gitlab::JiraImport::Stage::StartImportWorker.perform_async(project.id) - state.update(jid: job_id) if job_id + state.update(jid: job_id, scheduled_at: Time.now) if job_id end end @@ -97,4 +98,8 @@ class JiraImportState < ApplicationRecord } ) end + + def self.finished_imports_count + finished.sum(:imported_issues_count) + end end diff --git a/app/models/list.rb b/app/models/list.rb index 64247fdb983..ec211dfd497 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -3,8 +3,6 @@ class List < ApplicationRecord include Importable - prepend_if_ee('::EE::List') # rubocop: disable Cop/InjectEnterpriseEditionModule - belongs_to :board belongs_to :label has_many :list_user_preferences @@ -74,14 +72,18 @@ class List < ApplicationRecord label? ? label.name : list_type.humanize end + def collapsed?(user) + preferences = preferences_for(user) + + preferences.collapsed? + end + def as_json(options = {}) super(options).tap do |json| json[:collapsed] = false if options.key?(:collapsed) - preferences = preferences_for(options[:current_user]) - - json[:collapsed] = preferences.collapsed? + json[:collapsed] = collapsed?(options[:current_user]) end if options.key?(:label) @@ -100,3 +102,5 @@ class List < ApplicationRecord throw(:abort) unless destroyable? # rubocop:disable Cop/BanCatchThrow end end + +List.prepend_if_ee('::EE::List') diff --git a/app/models/member.rb b/app/models/member.rb index 5b33333aa23..791073da095 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Member < ApplicationRecord + include EachBatch include AfterCommitQueue include Sortable include Importable diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 68c51860c47..fa2e0cb8198 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -17,6 +17,11 @@ class ProjectMember < Member .where('projects.namespace_id in (?)', groups.select(:id)) end + scope :without_project_bots, -> do + left_join_users + .merge(User.without_project_bot) + end + class << self # Add users to projects with passed access option # diff --git a/app/models/members_preloader.rb b/app/models/members_preloader.rb index 1ed0434eacf..6da8d5f3161 100644 --- a/app/models/members_preloader.rb +++ b/app/models/members_preloader.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class MembersPreloader - prepend_if_ee('EE::MembersPreloader') # rubocop: disable Cop/InjectEnterpriseEditionModule - attr_reader :members def initialize(members) @@ -16,3 +14,5 @@ class MembersPreloader ActiveRecord::Associations::Preloader.new.preload(members.map(&:user), :u2f_registrations) end end + +MembersPreloader.prepend_if_ee('EE::MembersPreloader') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a28e054e13c..b4d0b729454 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -19,6 +19,7 @@ class MergeRequest < ApplicationRecord include ShaAttribute include IgnorableColumns include MilestoneEventable + include StateEventable sha_attribute :squash_commit_sha @@ -32,6 +33,7 @@ class MergeRequest < ApplicationRecord belongs_to :target_project, class_name: "Project" belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" + belongs_to :iteration, foreign_key: 'sprint_id' has_internal_id :iid, scope: :target_project, track_if: -> { !importing? }, init: ->(s) { s&.target_project&.merge_requests&.maximum(:iid) } @@ -864,7 +866,7 @@ class MergeRequest < ApplicationRecord check_service = MergeRequests::MergeabilityCheckService.new(self) - if async && Feature.enabled?(:async_merge_request_check_mergeability, project) + if async && Feature.enabled?(:async_merge_request_check_mergeability, project, default_enabled: true) check_service.async_execute else check_service.execute(retry_lease: false) @@ -873,7 +875,7 @@ class MergeRequest < ApplicationRecord # rubocop: enable CodeReuse/ServiceClass def diffable_merge_ref? - Feature.enabled?(:diff_compare_with_head, target_project) && can_be_merged? && merge_ref_head.present? + can_be_merged? && merge_ref_head.present? end # Returns boolean indicating the merge_status should be rechecked in order to @@ -1129,26 +1131,6 @@ class MergeRequest < ApplicationRecord end end - # Return array of possible target branches - # depends on target project of MR - def target_branches - if target_project.nil? - [] - else - target_project.repository.branch_names - end - end - - # Return array of possible source branches - # depends on source project of MR - def source_branches - if source_project.nil? - [] - else - source_project.repository.branch_names - end - end - def has_ci? return false if has_no_commits? @@ -1319,12 +1301,30 @@ class MergeRequest < ApplicationRecord compare_reports(Ci::CompareTestReportsService) end + def has_accessibility_reports? + return false unless Feature.enabled?(:accessibility_report_view, project) + + actual_head_pipeline.present? && actual_head_pipeline.has_reports?(Ci::JobArtifact.accessibility_reports) + end + def has_coverage_reports? return false unless Feature.enabled?(:coverage_report_view, project) actual_head_pipeline&.has_reports?(Ci::JobArtifact.coverage_reports) end + def has_terraform_reports? + actual_head_pipeline&.has_reports?(Ci::JobArtifact.terraform_reports) + end + + def compare_accessibility_reports + unless has_accessibility_reports? + return { status: :error, status_reason: _('This merge request does not have accessibility reports') } + end + + compare_reports(Ci::CompareAccessibilityReportsService) + end + # TODO: this method and compare_test_reports use the same # result type, which is handled by the controller's #reports_response. # we should minimize mistakes by isolating the common parts. @@ -1337,9 +1337,15 @@ class MergeRequest < ApplicationRecord compare_reports(Ci::GenerateCoverageReportsService) end - def has_exposed_artifacts? - return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true) + def find_terraform_reports + unless has_terraform_reports? + return { status: :error, status_reason: 'This merge request does not have terraform reports' } + end + compare_reports(Ci::GenerateTerraformReportsService) + end + + def has_exposed_artifacts? actual_head_pipeline&.has_exposed_artifacts? end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 7b15d21c095..f793bd3d76f 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -141,7 +141,7 @@ class MergeRequestDiff < ApplicationRecord after_create :save_git_content, unless: :importing? after_create_commit :set_as_latest_diff, unless: :importing? - after_save :update_external_diff_store, if: -> { !importing? && saved_change_to_external_diff? } + after_save :update_external_diff_store def self.find_by_diff_refs(diff_refs) find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha) @@ -385,34 +385,11 @@ class MergeRequestDiff < ApplicationRecord end end - # Carrierwave defines `write_uploader` dynamically on this class, so `super` - # does not work. Alias the carrierwave method so we can call it when needed - alias_method :carrierwave_write_uploader, :write_uploader - - # The `external_diff`, `external_diff_store`, and `stored_externally` - # columns were introduced in GitLab 11.8, but some background migration specs - # use factories that rely on current code with an old schema. Without these - # `has_attribute?` guards, they fail with a `MissingAttributeError`. - # - # For more details, see: https://gitlab.com/gitlab-org/gitlab-foss/issues/44990 - - def write_uploader(column, identifier) - carrierwave_write_uploader(column, identifier) if has_attribute?(column) - end - def update_external_diff_store - update_column(:external_diff_store, external_diff.object_store) if - has_attribute?(:external_diff_store) - end - - def saved_change_to_external_diff? - super if has_attribute?(:external_diff) - end + return unless saved_change_to_external_diff? || saved_change_to_stored_externally? - def stored_externally - super if has_attribute?(:stored_externally) + update_column(:external_diff_store, external_diff.object_store) end - alias_method :stored_externally?, :stored_externally # If enabled, yields the external file containing the diff. Otherwise, yields # nil. This method is not thread-safe, but it *is* re-entrant, which allows @@ -575,7 +552,6 @@ class MergeRequestDiff < ApplicationRecord end def use_external_diff? - return false unless has_attribute?(:external_diff) return false unless Gitlab.config.external_diffs.enabled case Gitlab.config.external_diffs.when diff --git a/app/models/metrics/users_starred_dashboard.rb b/app/models/metrics/users_starred_dashboard.rb new file mode 100644 index 00000000000..07748eb1431 --- /dev/null +++ b/app/models/metrics/users_starred_dashboard.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Metrics + class UsersStarredDashboard < ApplicationRecord + self.table_name = 'metrics_users_starred_dashboards' + + belongs_to :user, inverse_of: :metrics_users_starred_dashboards + belongs_to :project, inverse_of: :metrics_users_starred_dashboards + + validates :user_id, presence: true + validates :project_id, presence: true + validates :dashboard_path, presence: true, length: { maximum: 255 } + validates :dashboard_path, uniqueness: { scope: %i[user_id project_id] } + + scope :for_project, ->(project) { where(project: project) } + scope :for_project_dashboard, ->(project, path) { for_project(project).where(dashboard_path: path) } + end +end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 4ccfe314526..b5e4f62792e 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -1,88 +1,37 @@ # frozen_string_literal: true class Milestone < ApplicationRecord - # Represents a "No Milestone" state used for filtering Issues and Merge - # Requests that have no milestone assigned. - MilestoneStruct = Struct.new(:title, :name, :id) do - # Ensure these models match the interface required for exporting - def serializable_hash(_opts = {}) - { title: title, name: name, id: id } - end - end - - None = MilestoneStruct.new('No Milestone', 'No Milestone', 0) - Any = MilestoneStruct.new('Any Milestone', '', -1) - Upcoming = MilestoneStruct.new('Upcoming', '#upcoming', -2) - Started = MilestoneStruct.new('Started', '#started', -3) - - include CacheMarkdownField - include AtomicInternalId - include IidRoutes include Sortable include Referable - include StripAttribute + include Timebox include Milestoneish include FromUnion include Importable - include Gitlab::SQL::Pattern prepend_if_ee('::EE::Milestone') # rubocop: disable Cop/InjectEnterpriseEditionModule - cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description - - belongs_to :project - belongs_to :group - has_many :milestone_releases has_many :releases, through: :milestone_releases has_internal_id :iid, scope: :project, track_if: -> { !importing? }, init: ->(s) { s&.project&.milestones&.maximum(:iid) } has_internal_id :iid, scope: :group, track_if: -> { !importing? }, init: ->(s) { s&.group&.milestones&.maximum(:iid) } - has_many :issues - has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues - has_many :merge_requests has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - scope :of_projects, ->(ids) { where(project_id: ids) } - scope :of_groups, ->(ids) { where(group_id: ids) } scope :active, -> { with_state(:active) } - scope :closed, -> { with_state(:closed) } - scope :for_projects, -> { where(group: nil).includes(:project) } scope :started, -> { active.where('milestones.start_date <= CURRENT_DATE') } - - scope :for_projects_and_groups, -> (projects, groups) do - projects = projects.compact if projects.is_a? Array - projects = [] if projects.nil? - - groups = groups.compact if groups.is_a? Array - groups = [] if groups.nil? - - where(project_id: projects).or(where(group_id: groups)) - end - - scope :within_timeframe, -> (start_date, end_date) do - where('start_date is not NULL or due_date is not NULL') - .where('start_date is NULL or start_date <= ?', end_date) - .where('due_date is NULL or due_date >= ?', start_date) + scope :not_started, -> { active.where('milestones.start_date > CURRENT_DATE') } + scope :not_upcoming, -> do + active + .where('milestones.due_date <= CURRENT_DATE') + .order(:project_id, :group_id, :due_date) end scope :order_by_name_asc, -> { order(Arel::Nodes::Ascending.new(arel_table[:title].lower)) } scope :reorder_by_due_date_asc, -> { reorder(Gitlab::Database.nulls_last_order('due_date', 'ASC')) } - validates :group, presence: true, unless: :project - validates :project, presence: true, unless: :group - validates :title, presence: true - - validate :uniqueness_of_title, if: :title_changed? - validate :milestone_type_check - validate :start_date_should_be_less_than_due_date, if: proc { |m| m.start_date.present? && m.due_date.present? } - validate :dates_within_4_digits validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") } - strip_attributes :title - state_machine :state, initial: :active do event :close do transition active: :closed @@ -97,52 +46,6 @@ class Milestone < ApplicationRecord state :active end - alias_attribute :name, :title - - class << self - # Searches for milestones with a matching title or description. - # - # This method uses ILIKE on PostgreSQL and LIKE on MySQL. - # - # query - The search query as a String - # - # Returns an ActiveRecord::Relation. - def search(query) - fuzzy_search(query, [:title, :description]) - end - - # Searches for milestones with a matching title. - # - # This method uses ILIKE on PostgreSQL and LIKE on MySQL. - # - # query - The search query as a String - # - # Returns an ActiveRecord::Relation. - def search_title(query) - fuzzy_search(query, [:title]) - end - - def filter_by_state(milestones, state) - case state - when 'closed' then milestones.closed - when 'all' then milestones - else milestones.active - end - end - - def count_by_state - reorder(nil).group(:state).count - end - - def predefined_id?(id) - [Any.id, None.id, Upcoming.id, Started.id].include?(id) - end - - def predefined?(milestone) - predefined_id?(milestone&.id) - end - end - def self.reference_prefix '%' end @@ -220,7 +123,7 @@ class Milestone < ApplicationRecord end ## - # Returns the String necessary to reference this Milestone in Markdown. Group + # Returns the String necessary to reference a Milestone in Markdown. Group # milestones only support name references, and do not support cross-project # references. # @@ -248,10 +151,6 @@ class Milestone < ApplicationRecord self.class.reference_prefix + self.title end - def milestoneish_id - id - end - def for_display self end @@ -264,62 +163,24 @@ class Milestone < ApplicationRecord nil end - def title=(value) - write_attribute(:title, sanitize_title(value)) if value.present? - end + # TODO: remove after all code paths use `timebox_id` + # https://gitlab.com/gitlab-org/gitlab/-/issues/215688 + alias_method :milestoneish_id, :timebox_id + # TODO: remove after all code paths use (group|project)_timebox? + # https://gitlab.com/gitlab-org/gitlab/-/issues/215690 + alias_method :group_milestone?, :group_timebox? + alias_method :project_milestone?, :project_timebox? - def safe_title - title.to_slug.normalize.to_s - end - - def resource_parent - group || project - end - - def to_ability_name - model_name.singular - end - - def group_milestone? - group_id.present? - end - - def project_milestone? - project_id.present? - end - - def merge_requests_enabled? + def parent if group_milestone? - # Assume that groups have at least one project with merge requests enabled. - # Otherwise, we would need to load all of the projects from the database. - true - elsif project_milestone? - project&.merge_requests_enabled? + group + else + project end end private - # Milestone titles must be unique across project milestones and group milestones - def uniqueness_of_title - if project - relation = Milestone.for_projects_and_groups([project_id], [project.group&.id]) - elsif group - relation = Milestone.for_projects_and_groups(group.projects.select(:id), [group.id]) - end - - title_exists = relation.find_by_title(title) - errors.add(:title, _("already being used for another group or project milestone.")) if title_exists - end - - # Milestone should be either a project milestone or a group milestone - def milestone_type_check - if group_id && project_id - field = project_id_changed? ? :project_id : :group_id - errors.add(field, _("milestone should belong either to a project or a group.")) - end - end - def milestone_format_reference(format = :iid) raise ArgumentError, _('Unknown format') unless [:iid, :name].include?(format) @@ -334,26 +195,6 @@ class Milestone < ApplicationRecord end end - def sanitize_title(value) - CGI.unescape_html(Sanitize.clean(value.to_s)) - end - - def start_date_should_be_less_than_due_date - if due_date <= start_date - errors.add(:due_date, _("must be greater than start date")) - end - end - - def dates_within_4_digits - if start_date && start_date > Date.new(9999, 12, 31) - errors.add(:start_date, _("date must not be after 9999-12-31")) - end - - if due_date && due_date > Date.new(9999, 12, 31) - errors.add(:due_date, _("date must not be after 9999-12-31")) - end - end - def issues_finder_params { project_id: project_id, group_id: group_id, include_subgroups: group_id.present? }.compact end diff --git a/app/models/milestone_note.rb b/app/models/milestone_note.rb index 2ff9791feb0..19171e682b7 100644 --- a/app/models/milestone_note.rb +++ b/app/models/milestone_note.rb @@ -17,6 +17,6 @@ class MilestoneNote < SyntheticNote def note_text(html: false) format = milestone&.group_milestone? ? :name : :iid - milestone.nil? ? 'removed milestone' : "changed milestone to #{milestone.to_reference(project, format: format)}" + event.remove? ? 'removed milestone' : "changed milestone to #{milestone.to_reference(project, format: format)}" end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9e7589a1f18..8116f7a256f 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -14,6 +14,7 @@ class Namespace < ApplicationRecord include IgnorableColumns ignore_column :plan_id, remove_with: '13.1', remove_after: '2020-06-22' + ignore_column :trial_ends_on, remove_with: '13.2', remove_after: '2020-07-22' # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of @@ -135,11 +136,6 @@ class Namespace < ApplicationRecord name = host.delete_suffix(gitlab_host) Namespace.where(parent_id: nil).by_path(name) end - - # overridden in ee - def reset_ci_minutes!(namespace_id) - false - end end def default_branch_protection @@ -180,6 +176,10 @@ class Namespace < ApplicationRecord kind == 'user' end + def group? + type == 'Group' + end + def find_fork_of(project) return unless project.fork_network @@ -346,6 +346,21 @@ class Namespace < ApplicationRecord .try(name) end + def actual_plan + Plan.default + end + + def actual_limits + # We default to PlanLimits.new otherwise a lot of specs would fail + # On production each plan should already have associated limits record + # https://gitlab.com/gitlab-org/gitlab/issues/36037 + actual_plan.actual_limits + end + + def actual_plan_name + actual_plan.name + end + private def all_projects_with_pages diff --git a/app/models/namespace/root_storage_size.rb b/app/models/namespace/root_storage_size.rb new file mode 100644 index 00000000000..d61917e468e --- /dev/null +++ b/app/models/namespace/root_storage_size.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class Namespace::RootStorageSize + def initialize(root_namespace) + @root_namespace = root_namespace + end + + def above_size_limit? + return false if limit == 0 + + usage_ratio > 1 + end + + def usage_ratio + return 0 if limit == 0 + + current_size.to_f / limit.to_f + end + + def current_size + @current_size ||= root_namespace.root_storage_statistics&.storage_size + end + + def limit + @limit ||= Gitlab::CurrentSettings.namespace_storage_size_limit.megabytes + end + + private + + attr_reader :root_namespace +end diff --git a/app/models/note.rb b/app/models/note.rb index a2a711c987f..d174ba8fe83 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -159,6 +159,8 @@ class Note < ApplicationRecord after_save :touch_noteable, unless: :importing? after_destroy :expire_etag_cache after_save :store_mentions!, if: :any_mentionable_attributes_changed? + after_commit :notify_after_create, on: :create + after_commit :notify_after_destroy, on: :destroy class << self def model_name @@ -279,6 +281,10 @@ class Note < ApplicationRecord !for_personal_snippet? end + def for_design? + noteable_type == DesignManagement::Design.name + end + def for_issuable? for_issue? || for_merge_request? end @@ -505,6 +511,14 @@ class Note < ApplicationRecord noteable_object end + def notify_after_create + noteable&.after_note_created(self) + end + + def notify_after_destroy + noteable&.after_note_destroyed(self) + end + def banzai_render_context(field) super.merge(noteable: noteable, system_note: system?) end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 486da2c6b45..da5e4012f05 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -2,6 +2,7 @@ class PagesDomain < ApplicationRecord include Presentable + include FromUnion VERIFICATION_KEY = 'gitlab-pages-verification-code' VERIFICATION_THRESHOLD = 3.days.freeze @@ -58,12 +59,14 @@ class PagesDomain < ApplicationRecord end scope :need_auto_ssl_renewal, -> do - expiring = where(certificate_valid_not_after: nil).or( - where(arel_table[:certificate_valid_not_after].lt(SSL_RENEWAL_THRESHOLD.from_now))) + enabled_and_not_failed = where(auto_ssl_enabled: true, auto_ssl_failed: false) - user_provided_or_expiring = certificate_user_provided.or(expiring) + user_provided = enabled_and_not_failed.certificate_user_provided + certificate_not_valid = enabled_and_not_failed.where(certificate_valid_not_after: nil) + certificate_expiring = enabled_and_not_failed + .where(arel_table[:certificate_valid_not_after].lt(SSL_RENEWAL_THRESHOLD.from_now)) - where(auto_ssl_enabled: true).merge(user_provided_or_expiring) + from_union([user_provided, certificate_not_valid, certificate_expiring]) end scope :for_removal, -> { where("remove_at < ?", Time.now) } diff --git a/app/models/performance_monitoring/prometheus_dashboard.rb b/app/models/performance_monitoring/prometheus_dashboard.rb index 30fb1935a27..57222c61b36 100644 --- a/app/models/performance_monitoring/prometheus_dashboard.rb +++ b/app/models/performance_monitoring/prometheus_dashboard.rb @@ -4,7 +4,7 @@ module PerformanceMonitoring class PrometheusDashboard include ActiveModel::Model - attr_accessor :dashboard, :panel_groups, :path, :environment, :priority + attr_accessor :dashboard, :panel_groups, :path, :environment, :priority, :templating validates :dashboard, presence: true validates :panel_groups, presence: true diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index af079f7ebc4..7afee2a35cb 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -4,6 +4,7 @@ class PersonalAccessToken < ApplicationRecord include Expirable include TokenAuthenticatable include Sortable + extend ::Gitlab::Utils::Override add_authentication_token_field :token, digest: true @@ -23,6 +24,8 @@ class PersonalAccessToken < ApplicationRecord scope :without_impersonation, -> { where(impersonation: false) } scope :for_user, -> (user) { where(user: user) } scope :preload_users, -> { preload(:user) } + scope :order_expires_at_asc, -> { reorder(expires_at: :asc) } + scope :order_expires_at_desc, -> { reorder(expires_at: :desc) } validates :scopes, presence: true validate :validate_scopes @@ -39,12 +42,14 @@ class PersonalAccessToken < ApplicationRecord def self.redis_getdel(user_id) Gitlab::Redis::SharedState.with do |redis| - encrypted_token = redis.get(redis_shared_state_key(user_id)) - redis.del(redis_shared_state_key(user_id)) + redis_key = redis_shared_state_key(user_id) + encrypted_token = redis.get(redis_key) + redis.del(redis_key) + begin Gitlab::CryptoHelper.aes256_gcm_decrypt(encrypted_token) rescue => ex - logger.warn "Failed to decrypt PersonalAccessToken value stored in Redis for User ##{user_id}: #{ex.class}" + logger.warn "Failed to decrypt #{self.name} value stored in Redis for key ##{redis_key}: #{ex.class}" encrypted_token end end @@ -58,6 +63,16 @@ class PersonalAccessToken < ApplicationRecord end end + override :simple_sorts + def self.simple_sorts + super.merge( + { + 'expires_at_asc' => -> { order_expires_at_asc }, + 'expires_at_desc' => -> { order_expires_at_desc } + } + ) + end + protected def validate_scopes diff --git a/app/models/personal_snippet.rb b/app/models/personal_snippet.rb index 1b5be8698b1..197795dccfe 100644 --- a/app/models/personal_snippet.rb +++ b/app/models/personal_snippet.rb @@ -2,4 +2,8 @@ class PersonalSnippet < Snippet include WithUploads + + def skip_project_check? + true + end end diff --git a/app/models/plan.rb b/app/models/plan.rb new file mode 100644 index 00000000000..acac5f9aeae --- /dev/null +++ b/app/models/plan.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class Plan < ApplicationRecord + DEFAULT = 'default'.freeze + + has_one :limits, class_name: 'PlanLimits' + + ALL_PLANS = [DEFAULT].freeze + DEFAULT_PLANS = [DEFAULT].freeze + private_constant :ALL_PLANS, :DEFAULT_PLANS + + # This always returns an object + def self.default + Gitlab::SafeRequestStore.fetch(:plan_default) do + # find_by allows us to find object (cheaply) against replica DB + # safe_find_or_create_by does stick to primary DB + find_by(name: DEFAULT) || safe_find_or_create_by(name: DEFAULT) + end + end + + def self.all_plans + ALL_PLANS + end + + def self.default_plans + DEFAULT_PLANS + end + + def actual_limits + self.limits || PlanLimits.new + end + + def default? + self.class.default_plans.include?(name) + end + + def paid? + false + end +end + +Plan.prepend_if_ee('EE::Plan') diff --git a/app/models/plan_limits.rb b/app/models/plan_limits.rb new file mode 100644 index 00000000000..575105cfd79 --- /dev/null +++ b/app/models/plan_limits.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class PlanLimits < ApplicationRecord + belongs_to :plan + + def exceeded?(limit_name, object) + return false unless enabled?(limit_name) + + if object.is_a?(Integer) + object >= read_attribute(limit_name) + else + # object.count >= limit value is slower than checking + # if a record exists at the limit value - 1 position. + object.offset(read_attribute(limit_name) - 1).exists? + end + end + + private + + def enabled?(limit_name) + read_attribute(limit_name) > 0 + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 5db349463d8..c0dd2eb8584 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -3,6 +3,7 @@ require 'carrierwave/orm/activerecord' class Project < ApplicationRecord + extend ::Gitlab::Utils::Override include Gitlab::ConfigHelper include Gitlab::VisibilityLevel include AccessRequestable @@ -18,6 +19,7 @@ class Project < ApplicationRecord include SelectForProjectAuthorization include Presentable include HasRepository + include HasWiki include Routable include GroupDescendant include Gitlab::SQL::Pattern @@ -175,6 +177,7 @@ class Project < ApplicationRecord has_one :packagist_service has_one :hangouts_chat_service has_one :unify_circuit_service + has_one :webex_teams_service has_one :root_of_fork_network, foreign_key: 'root_project_id', @@ -206,12 +209,14 @@ class Project < ApplicationRecord has_many :services has_many :events has_many :milestones + has_many :iterations has_many :notes has_many :snippets, class_name: 'ProjectSnippet' has_many :hooks, class_name: 'ProjectHook' has_many :protected_branches has_many :protected_tags has_many :repository_languages, -> { order "share DESC" } + has_many :designs, inverse_of: :project, class_name: 'DesignManagement::Design' has_many :project_authorizations has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' @@ -254,6 +259,9 @@ class Project < ApplicationRecord has_many :prometheus_alerts, inverse_of: :project has_many :prometheus_alert_events, inverse_of: :project has_many :self_managed_prometheus_alert_events, inverse_of: :project + has_many :metrics_users_starred_dashboards, class_name: 'Metrics::UsersStarredDashboard', inverse_of: :project + + has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :project # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy @@ -295,6 +303,7 @@ class Project < ApplicationRecord has_many :project_deploy_tokens has_many :deploy_tokens, through: :project_deploy_tokens has_many :resource_groups, class_name: 'Ci::ResourceGroup', inverse_of: :project + has_many :freeze_periods, class_name: 'Ci::FreezePeriod', inverse_of: :project has_one :auto_devops, class_name: 'ProjectAutoDevops', inverse_of: :project, autosave: true has_many :custom_attributes, class_name: 'ProjectCustomAttribute' @@ -315,10 +324,13 @@ class Project < ApplicationRecord has_many :import_failures, inverse_of: :project has_many :jira_imports, -> { order 'jira_imports.created_at' }, class_name: 'JiraImportState', inverse_of: :project - has_many :daily_report_results, class_name: 'Ci::DailyReportResult' + has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult' + + has_many :repository_storage_moves, class_name: 'ProjectRepositoryStorageMove' accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true + accepts_nested_attributes_for :project_setting, update_only: true accepts_nested_attributes_for :import_data accepts_nested_attributes_for :auto_devops, update_only: true accepts_nested_attributes_for :ci_cd_settings, update_only: true @@ -342,6 +354,9 @@ class Project < ApplicationRecord :wiki_access_level, :snippets_access_level, :builds_access_level, :repository_access_level, :pages_access_level, :metrics_dashboard_access_level, to: :project_feature, allow_nil: true + delegate :show_default_award_emojis, :show_default_award_emojis=, + :show_default_award_emojis?, + to: :project_setting, allow_nil: true delegate :scheduled?, :started?, :in_progress?, :failed?, :finished?, prefix: :import, to: :import_state, allow_nil: true delegate :no_import?, to: :import_state, allow_nil: true @@ -355,6 +370,7 @@ class Project < ApplicationRecord delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings, prefix: :ci delegate :forward_deployment_enabled, :forward_deployment_enabled=, :forward_deployment_enabled?, to: :ci_cd_settings + delegate :actual_limits, :actual_plan_name, to: :namespace, allow_nil: true # Validations validates :creator, presence: true, on: :create @@ -386,7 +402,6 @@ class Project < ApplicationRecord validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } validate :visibility_level_allowed_by_group, if: :should_validate_visibility_level? validate :visibility_level_allowed_as_fork, if: :should_validate_visibility_level? - validate :check_wiki_path_conflict validate :validate_pages_https_only, if: -> { changes.has_key?(:pages_https_only) } validates :repository_storage, presence: true, @@ -515,12 +530,14 @@ class Project < ApplicationRecord def self.public_or_visible_to_user(user = nil, min_access_level = nil) min_access_level = nil if user&.admin? - if user + return public_to_user unless user + + if user.is_a?(DeployToken) + user.projects + else where('EXISTS (?) OR projects.visibility_level IN (?)', user.authorizations_for_projects(min_access_level: min_access_level), Gitlab::VisibilityLevel.levels_for_user(user)) - else - public_to_user end end @@ -785,6 +802,11 @@ class Project < ApplicationRecord Feature.enabled?(:jira_issue_import, self, default_enabled: true) end + # LFS and hashed repository storage are required for using Design Management. + def design_management_enabled? + lfs_enabled? && hashed_storage?(:repository) + end + def team @team ||= ProjectTeam.new(self) end @@ -793,6 +815,12 @@ class Project < ApplicationRecord @repository ||= Repository.new(full_path, self, shard: repository_storage, disk_path: disk_path) end + def design_repository + strong_memoize(:design_repository) do + DesignManagement::Repository.new(self) + end + end + def cleanup @repository = nil end @@ -819,7 +847,7 @@ class Project < ApplicationRecord latest_pipeline = ci_pipelines.latest_successful_for_ref(ref) return unless latest_pipeline - latest_pipeline.builds.latest.with_artifacts_archive.find_by(name: job_name) + latest_pipeline.builds.latest.with_downloadable_artifacts.find_by(name: job_name) end def latest_successful_build_for_sha(job_name, sha) @@ -828,7 +856,7 @@ class Project < ApplicationRecord latest_pipeline = ci_pipelines.latest_successful_for_sha(sha) return unless latest_pipeline - latest_pipeline.builds.latest.with_artifacts_archive.find_by(name: job_name) + latest_pipeline.builds.latest.with_downloadable_artifacts.find_by(name: job_name) end def latest_successful_build_for_ref!(job_name, ref = default_branch) @@ -865,10 +893,12 @@ class Project < ApplicationRecord raise Projects::ImportService::Error, _('Jira import feature is disabled.') unless jira_issues_import_feature_flag_enabled? raise Projects::ImportService::Error, _('Jira integration not configured.') unless jira_service&.active? - return unless user + if user + raise Projects::ImportService::Error, _('Cannot import because issues are not available in this project.') unless feature_available?(:issues, user) + raise Projects::ImportService::Error, _('You do not have permissions to run the import.') unless user.can?(:admin_project, self) + end - raise Projects::ImportService::Error, _('Cannot import because issues are not available in this project.') unless feature_available?(:issues, user) - raise Projects::ImportService::Error, _('You do not have permissions to run the import.') unless user.can?(:admin_project, self) + raise Projects::ImportService::Error, _('Unable to connect to the Jira instance. Please check your Jira integration configuration.') unless jira_service.test(nil)[:success] end def human_import_status_name @@ -1056,16 +1086,6 @@ class Project < ApplicationRecord self.errors.add(:visibility_level, _("%{level_name} is not allowed since the fork source project has lower visibility.") % { level_name: level_name }) end - def check_wiki_path_conflict - return if path.blank? - - path_to_check = path.ends_with?('.wiki') ? path.chomp('.wiki') : "#{path}.wiki" - - if Project.where(namespace_id: namespace_id, path: path_to_check).exists? - errors.add(:name, _('has already been taken')) - end - end - def pages_https_only return false unless Gitlab.config.pages.external_https @@ -1179,11 +1199,7 @@ class Project < ApplicationRecord end def issues_tracker - if external_issue_tracker - external_issue_tracker - else - default_issue_tracker - end + external_issue_tracker || default_issue_tracker end def external_issue_reference_pattern @@ -1328,11 +1344,7 @@ class Project < ApplicationRecord # rubocop: enable CodeReuse/ServiceClass def owner - if group - group - else - namespace.try(:owner) - end + group || namespace.try(:owner) end def to_ability_name @@ -1432,15 +1444,12 @@ class Project < ApplicationRecord # Expires various caches before a project is renamed. def expire_caches_before_rename(old_path) - repo = Repository.new(old_path, self, shard: repository_storage) - wiki = Repository.new("#{old_path}.wiki", self, shard: repository_storage, repo_type: Gitlab::GlRepository::WIKI) + project_repo = Repository.new(old_path, self, shard: repository_storage) + wiki_repo = Repository.new("#{old_path}#{Gitlab::GlRepository::WIKI.path_suffix}", self, shard: repository_storage, repo_type: Gitlab::GlRepository::WIKI) + design_repo = Repository.new("#{old_path}#{Gitlab::GlRepository::DESIGN.path_suffix}", self, shard: repository_storage, repo_type: Gitlab::GlRepository::DESIGN) - if repo.exists? - repo.before_delete - end - - if wiki.exists? - wiki.before_delete + [project_repo, wiki_repo, design_repo].each do |repo| + repo.before_delete if repo.exists? end end @@ -1517,6 +1526,10 @@ class Project < ApplicationRecord end end + def bots + users.project_bot + end + # Filters `users` to return only authorized users of the project def members_among(users) if users.is_a?(ActiveRecord::Relation) && !users.loaded? @@ -1565,10 +1578,6 @@ class Project < ApplicationRecord create_repository(force: true) unless repository_exists? end - def wiki_repository_exists? - wiki.repository_exists? - end - # update visibility_level of forks def update_forks_visibility_level return if unlink_forks_upon_visibility_decrease_enabled? @@ -1582,20 +1591,6 @@ class Project < ApplicationRecord end end - def create_wiki - ProjectWiki.new(self, self.owner).wiki - true - rescue ProjectWiki::CouldNotCreateWikiError - errors.add(:base, _('Failed create wiki')) - false - end - - def wiki - strong_memoize(:wiki) do - ProjectWiki.new(self, self.owner) - end - end - def allowed_to_share_with_group? !namespace.share_with_group_lock end @@ -2024,6 +2019,14 @@ class Project < ApplicationRecord end end + def ci_instance_variables_for(ref:) + if protected_for?(ref) + Ci::InstanceVariable.all_cached + else + Ci::InstanceVariable.unprotected_cached + end + end + def protected_for?(ref) raise Repository::AmbiguousRefError if repository.ambiguous_ref?(ref) @@ -2085,7 +2088,12 @@ class Project < ApplicationRecord raise ArgumentError unless ::Gitlab.config.repositories.storages.key?(new_repository_storage_key) - run_after_commit { ProjectUpdateRepositoryStorageWorker.perform_async(id, new_repository_storage_key) } + storage_move = repository_storage_moves.create!( + source_storage_name: repository_storage, + destination_storage_name: new_repository_storage_key + ) + storage_move.schedule! + self.repository_read_only = true end @@ -2425,6 +2433,11 @@ class Project < ApplicationRecord jira_imports.last end + override :after_wiki_activity + def after_wiki_activity + touch(:last_activity_at, :last_repository_updated_at) + end + private def find_service(services, name) diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index e81d9d0f5fe..366852d93bf 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -2,7 +2,6 @@ class ProjectAuthorization < ApplicationRecord include FromUnion - prepend_if_ee('::EE::ProjectAuthorization') # rubocop: disable Cop/InjectEnterpriseEditionModule belongs_to :user belongs_to :project @@ -30,3 +29,5 @@ class ProjectAuthorization < ApplicationRecord end end end + +ProjectAuthorization.prepend_if_ee('::EE::ProjectAuthorization') diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 39e177e8bd8..c295837002a 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -37,8 +37,6 @@ class ProjectCiCdSetting < ApplicationRecord private def set_default_git_depth - return unless Feature.enabled?(:ci_set_project_default_git_depth, default_enabled: true) - self.default_git_depth ||= DEFAULT_GIT_DEPTH end end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 31a3fa12c00..9201cd24d66 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -23,7 +23,7 @@ class ProjectFeature < ApplicationRecord PUBLIC = 30 FEATURES = %i(issues forking merge_requests wiki snippets builds repository pages metrics_dashboard).freeze - PRIVATE_FEATURES_MIN_ACCESS_LEVEL = { merge_requests: Gitlab::Access::REPORTER }.freeze + PRIVATE_FEATURES_MIN_ACCESS_LEVEL = { merge_requests: Gitlab::Access::REPORTER, metrics_dashboard: Gitlab::Access::REPORTER }.freeze PRIVATE_FEATURES_MIN_ACCESS_LEVEL_FOR_PRIVATE_PROJECT = { repository: Gitlab::Access::REPORTER }.freeze STRING_OPTIONS = HashWithIndifferentAccess.new({ 'disabled' => DISABLED, diff --git a/app/models/project_repository_storage_move.rb b/app/models/project_repository_storage_move.rb new file mode 100644 index 00000000000..e88cc5cfca6 --- /dev/null +++ b/app/models/project_repository_storage_move.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# ProjectRepositoryStorageMove are details of repository storage moves for a +# project. For example, moving a project to another gitaly node to help +# balance storage capacity. +class ProjectRepositoryStorageMove < ApplicationRecord + include AfterCommitQueue + + belongs_to :project, inverse_of: :repository_storage_moves + + validates :project, presence: true + validates :state, presence: true + validates :source_storage_name, + on: :create, + presence: true, + inclusion: { in: ->(_) { Gitlab.config.repositories.storages.keys } } + validates :destination_storage_name, + on: :create, + presence: true, + inclusion: { in: ->(_) { Gitlab.config.repositories.storages.keys } } + + state_machine initial: :initial do + event :schedule do + transition initial: :scheduled + end + + event :start do + transition scheduled: :started + end + + event :finish do + transition started: :finished + end + + event :do_fail do + transition [:initial, :scheduled, :started] => :failed + end + + after_transition initial: :scheduled do |storage_move, _| + storage_move.run_after_commit do + ProjectUpdateRepositoryStorageWorker.perform_async( + storage_move.project_id, + storage_move.destination_storage_name, + storage_move.id + ) + end + end + + state :initial, value: 1 + state :scheduled, value: 2 + state :started, value: 3 + state :finished, value: 4 + state :failed, value: 5 + end + + scope :order_created_at_desc, -> { order(created_at: :desc) } + scope :with_projects, -> { includes(project: :route) } +end diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index dc62a4c8908..0a2d9120adc 100644 --- a/app/models/project_services/chat_message/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -2,8 +2,6 @@ module ChatMessage class MergeMessage < BaseMessage - prepend_if_ee('::EE::ChatMessage::MergeMessage') # rubocop: disable Cop/InjectEnterpriseEditionModule - attr_reader :merge_request_iid attr_reader :source_branch attr_reader :target_branch @@ -71,3 +69,5 @@ module ChatMessage end end end + +ChatMessage::MergeMessage.prepend_if_ee('::EE::ChatMessage::MergeMessage') diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 50b982a803f..1cd3837433f 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -52,8 +52,6 @@ module ChatMessage def attachments return message if markdown - return [{ text: format(message), color: attachment_color }] unless fancy_notifications? - [{ fallback: format(message), color: attachment_color, @@ -103,10 +101,6 @@ module ChatMessage failed_jobs.uniq { |job| job[:name] }.reverse end - def fancy_notifications? - Feature.enabled?(:fancy_pipeline_slack_notifications, default_enabled: true) - end - def failed_stages_field { title: s_("ChatMessage|Failed stage").pluralize(failed_stages.length), @@ -166,42 +160,22 @@ module ChatMessage end def humanized_status - if fancy_notifications? - case status - when 'success' - detailed_status == "passed with warnings" ? s_("ChatMessage|has passed with warnings") : s_("ChatMessage|has passed") - when 'failed' - s_("ChatMessage|has failed") - else - status - end + case status + when 'success' + detailed_status == "passed with warnings" ? s_("ChatMessage|has passed with warnings") : s_("ChatMessage|has passed") + when 'failed' + s_("ChatMessage|has failed") else - case status - when 'success' - s_("ChatMessage|passed") - when 'failed' - s_("ChatMessage|failed") - else - status - end + status end end def attachment_color - if fancy_notifications? - case status - when 'success' - detailed_status == 'passed with warnings' ? 'warning' : 'good' - else - 'danger' - end + case status + when 'success' + detailed_status == 'passed with warnings' ? 'warning' : 'good' else - case status - when 'success' - 'good' - else - 'danger' - end + 'danger' end end @@ -230,7 +204,7 @@ module ChatMessage end def pipeline_url - if fancy_notifications? && failed_jobs.any? + if failed_jobs.any? pipeline_failed_jobs_url else "#{project_url}/pipelines/#{pipeline_id}" diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index eaddac9cce3..53da874ede8 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -25,6 +25,11 @@ class JiraService < IssueTrackerService before_update :reset_password + enum comment_detail: { + standard: 1, + all_details: 2 + } + alias_method :project_url, :url # When these are false GitLab does not create cross reference @@ -172,6 +177,7 @@ class JiraService < IssueTrackerService noteable_id = noteable.respond_to?(:iid) ? noteable.iid : noteable.id noteable_type = noteable_name(noteable) entity_url = build_entity_url(noteable_type, noteable_id) + entity_meta = build_entity_meta(noteable) data = { user: { @@ -180,12 +186,15 @@ class JiraService < IssueTrackerService }, project: { name: project.full_path, - url: resource_url(namespace_project_path(project.namespace, project)) # rubocop:disable Cop/ProjectPathHelper + url: resource_url(project_path(project)) }, entity: { + id: entity_meta[:id], name: noteable_type.humanize.downcase, url: entity_url, - title: noteable.title + title: noteable.title, + description: entity_meta[:description], + branch: entity_meta[:branch] } } @@ -259,14 +268,11 @@ class JiraService < IssueTrackerService end def add_comment(data, issue) - user_name = data[:user][:name] - user_url = data[:user][:url] entity_name = data[:entity][:name] entity_url = data[:entity][:url] entity_title = data[:entity][:title] - project_name = data[:project][:name] - message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title.chomp}'" + message = comment_message(data) link_title = "#{entity_name.capitalize} - #{entity_title}" link_props = build_remote_link_props(url: entity_url, title: link_title) @@ -275,6 +281,37 @@ class JiraService < IssueTrackerService end end + def comment_message(data) + user_link = build_jira_link(data[:user][:name], data[:user][:url]) + + entity = data[:entity] + entity_ref = all_details? ? "#{entity[:name]} #{entity[:id]}" : "a #{entity[:name]}" + entity_link = build_jira_link(entity_ref, entity[:url]) + + project_link = build_jira_link(project.full_name, Gitlab::Routing.url_helpers.project_url(project)) + branch = + if entity[:branch].present? + s_('JiraService| on branch %{branch_link}') % { + branch_link: build_jira_link(entity[:branch], project_tree_url(project, entity[:branch])) + } + end + + entity_message = entity[:description].presence if all_details? + entity_message ||= entity[:title].chomp + + s_('JiraService|%{user_link} mentioned this issue in %{entity_link} of %{project_link}%{branch}:{quote}%{entity_message}{quote}') % { + user_link: user_link, + entity_link: entity_link, + project_link: project_link, + branch: branch, + entity_message: entity_message + } + end + + def build_jira_link(title, url) + "[#{title}|#{url}]" + end + def has_resolution?(issue) issue.respond_to?(:resolution) && issue.resolution.present? end @@ -348,6 +385,23 @@ class JiraService < IssueTrackerService ) end + def build_entity_meta(noteable) + if noteable.is_a?(Commit) + { + id: noteable.short_id, + description: noteable.safe_message, + branch: noteable.ref_names(project.repository).first + } + elsif noteable.is_a?(MergeRequest) + { + id: noteable.to_reference, + branch: noteable.source_branch + } + else + {} + end + end + def noteable_name(noteable) name = noteable.model_name.singular diff --git a/app/models/project_services/mattermost_slash_commands_service.rb b/app/models/project_services/mattermost_slash_commands_service.rb index ca324f68d2d..0fd85e3a5a9 100644 --- a/app/models/project_services/mattermost_slash_commands_service.rb +++ b/app/models/project_services/mattermost_slash_commands_service.rb @@ -36,6 +36,10 @@ class MattermostSlashCommandsService < SlashCommandsService [[], e.message] end + def chat_responder + ::Gitlab::Chat::Responder::Mattermost + end + private def command(params) diff --git a/app/models/project_services/mock_monitoring_service.rb b/app/models/project_services/mock_monitoring_service.rb index bcf8f1df5da..25ae0f6b60d 100644 --- a/app/models/project_services/mock_monitoring_service.rb +++ b/app/models/project_services/mock_monitoring_service.rb @@ -14,7 +14,7 @@ class MockMonitoringService < MonitoringService end def metrics(environment) - JSON.parse(File.read(Rails.root + 'spec/fixtures/metrics.json')) + Gitlab::Json.parse(File.read(Rails.root + 'spec/fixtures/metrics.json')) end def can_test? diff --git a/app/models/project_services/webex_teams_service.rb b/app/models/project_services/webex_teams_service.rb new file mode 100644 index 00000000000..1d791b19486 --- /dev/null +++ b/app/models/project_services/webex_teams_service.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +class WebexTeamsService < ChatNotificationService + def title + 'Webex Teams' + end + + def description + 'Receive event notifications in Webex Teams' + end + + def self.to_param + 'webex_teams' + end + + def help + 'This service sends notifications about projects events to a Webex Teams conversation.<br /> + To set up this service: + <ol> + <li><a href="https://apphub.webex.com/teams/applications/incoming-webhooks-cisco-systems">Set up an incoming webhook for your conversation</a>. All notifications will come to this conversation.</li> + <li>Paste the <strong>Webhook URL</strong> into the field below.</li> + <li>Select events below to enable notifications.</li> + </ol>' + end + + def event_field(event) + end + + def default_channel_placeholder + end + + def self.supported_events + %w[push issue confidential_issue merge_request note confidential_note tag_push + pipeline wiki_page] + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "e.g. https://api.ciscospark.com/v1/webhooks/incoming/…", required: true }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + ] + end + + private + + def notify(message, opts) + header = { 'Content-Type' => 'application/json' } + response = Gitlab::HTTP.post(webhook, headers: header, body: { markdown: message.pretext }.to_json) + + response if response.success? + end + + def custom_data(data) + super(data).merge(markdown: true) + end +end diff --git a/app/models/project_services/youtrack_service.rb b/app/models/project_services/youtrack_service.rb index 0815e27850d..40203ad692d 100644 --- a/app/models/project_services/youtrack_service.rb +++ b/app/models/project_services/youtrack_service.rb @@ -27,8 +27,8 @@ class YoutrackService < IssueTrackerService def fields [ { type: 'text', name: 'description', placeholder: description }, - { type: 'text', name: 'project_url', placeholder: 'Project url', required: true }, - { type: 'text', name: 'issues_url', placeholder: 'Issue url', required: true } + { type: 'text', name: 'project_url', title: 'Project URL', placeholder: 'Project URL', required: true }, + { type: 'text', name: 'issues_url', title: 'Issue URL', placeholder: 'Issue URL', required: true } ] end end diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index b71ed75dde6..6f04a36392d 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -21,6 +21,9 @@ class ProjectStatistics < ApplicationRecord scope :for_project_ids, ->(project_ids) { where(project_id: project_ids) } + scope :for_namespaces, -> (namespaces) { where(namespace: namespaces) } + scope :with_any_ci_minutes_used, -> { where.not(shared_runners_seconds: 0) } + def total_repository_size repository_size + lfs_objects_size end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 708b45cf5f0..5df0a33dc9a 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -1,219 +1,17 @@ # frozen_string_literal: true -class ProjectWiki - include Storage::LegacyProjectWiki - include Gitlab::Utils::StrongMemoize +class ProjectWiki < Wiki + alias_method :project, :container - MARKUPS = { - 'Markdown' => :markdown, - 'RDoc' => :rdoc, - 'AsciiDoc' => :asciidoc, - 'Org' => :org - }.freeze unless defined?(MARKUPS) + # Project wikis are tied to the main project storage + delegate :storage, :repository_storage, :hashed_storage?, to: :container - CouldNotCreateWikiError = Class.new(StandardError) - SIDEBAR = '_sidebar' - - TITLE_ORDER = 'title' - CREATED_AT_ORDER = 'created_at' - DIRECTION_DESC = 'desc' - DIRECTION_ASC = 'asc' - - attr_reader :project, :user - - # Returns a string describing what went wrong after - # an operation fails. - attr_reader :error_message - - def initialize(project, user = nil) - @project = project - @user = user - end - - delegate :repository_storage, :hashed_storage?, to: :project - - def path - @project.path + '.wiki' - end - - def full_path - @project.full_path + '.wiki' - end - alias_method :id, :full_path - - # @deprecated use full_path when you need it for an URL route or disk_path when you want to point to the filesystem - alias_method :path_with_namespace, :full_path - - def web_url(only_path: nil) - Gitlab::UrlBuilder.build(self, only_path: only_path) - end - - def url_to_repo - ssh_url_to_repo - end - - def ssh_url_to_repo - Gitlab::RepositoryUrlBuilder.build(repository.full_path, protocol: :ssh) - end - - def http_url_to_repo - Gitlab::RepositoryUrlBuilder.build(repository.full_path, protocol: :http) - end - - def wiki_base_path - [Gitlab.config.gitlab.relative_url_root, '/', @project.full_path, '/-', '/wikis'].join('') - end - - # Returns the Gitlab::Git::Wiki object. - def wiki - strong_memoize(:wiki) do - repository.create_if_not_exists - raise CouldNotCreateWikiError unless repository_exists? - - Gitlab::Git::Wiki.new(repository.raw) - end - rescue => err - Gitlab::ErrorTracking.track_exception(err, project_wiki: { project_id: project.id, full_path: full_path, disk_path: disk_path }) - raise CouldNotCreateWikiError - end - - def repository_exists? - !!repository.exists? - end - - def has_home_page? - !!find_page('home') - end - - def empty? - list_pages(limit: 1).empty? - end - - def exists? - !empty? - end - - # Lists wiki pages of the repository. - # - # limit - max number of pages returned by the method. - # sort - criterion by which the pages are sorted. - # direction - order of the sorted pages. - # load_content - option, which specifies whether the content inside the page - # will be loaded. - # - # Returns an Array of GitLab WikiPage instances or an - # empty Array if this Wiki has no pages. - def list_pages(limit: 0, sort: nil, direction: DIRECTION_ASC, load_content: false) - wiki.list_pages( - limit: limit, - sort: sort, - direction_desc: direction == DIRECTION_DESC, - load_content: load_content - ).map do |page| - WikiPage.new(self, page) - end - end - - # Finds a page within the repository based on a tile - # or slug. - # - # title - The human readable or parameterized title of - # the page. - # - # Returns an initialized WikiPage instance or nil - def find_page(title, version = nil) - page_title, page_dir = page_title_and_dir(title) - - if page = wiki.page(title: page_title, version: version, dir: page_dir) - WikiPage.new(self, page) - end - end - - def find_sidebar(version = nil) - find_page(SIDEBAR, version) - end - - def find_file(name, version = nil) - wiki.file(name, version) - end - - def create_page(title, content, format = :markdown, message = nil) - commit = commit_details(:created, message, title) - - wiki.write_page(title, format.to_sym, content, commit) - - update_project_activity - rescue Gitlab::Git::Wiki::DuplicatePageError => e - @error_message = "Duplicate page: #{e.message}" - false - end - - def update_page(page, content:, title: nil, format: :markdown, message: nil) - commit = commit_details(:updated, message, page.title) - - wiki.update_page(page.path, title || page.name, format.to_sym, content, commit) - - update_project_activity - end - - def delete_page(page, message = nil) - return unless page - - wiki.delete_page(page.path, commit_details(:deleted, message, page.title)) - - update_project_activity - end - - def page_title_and_dir(title) - return unless title - - title_array = title.split("/") - title = title_array.pop - [title, title_array.join("/")] - end - - def repository - @repository ||= Repository.new(full_path, @project, shard: repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::WIKI) - end - - def default_branch - wiki.class.default_ref - end - - def ensure_repository - raise CouldNotCreateWikiError unless wiki.repository_exists? - end - - def hook_attrs - { - web_url: web_url, - git_ssh_url: ssh_url_to_repo, - git_http_url: http_url_to_repo, - path_with_namespace: full_path, - default_branch: default_branch - } - end - - private - - def commit_details(action, message = nil, title = nil) - commit_message = message.presence || default_message(action, title) - git_user = Gitlab::Git::User.from_gitlab(user) - - Gitlab::Git::Wiki::CommitDetails.new(user.id, - git_user.username, - git_user.name, - git_user.email, - commit_message) - end - - def default_message(action, title) - "#{user.username} #{action} page: #{title}" - end - - def update_project_activity - @project.touch(:last_activity_at, :last_repository_updated_at) + override :disk_path + def disk_path(*args, &block) + container.disk_path + '.wiki' end end +# TODO: Remove this once we implement ES support for group wikis. +# https://gitlab.com/gitlab-org/gitlab/-/issues/207889 ProjectWiki.prepend_if_ee('EE::ProjectWiki') diff --git a/app/models/release.rb b/app/models/release.rb index 403087a2cad..a0245105cd9 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -34,8 +34,6 @@ class Release < ApplicationRecord delegate :repository, to: :project - after_commit :notify_new_release, on: :create, unless: :importing? - MAX_NUMBER_TO_DISPLAY = 3 def to_param @@ -81,14 +79,6 @@ class Release < ApplicationRecord self.milestones.map {|m| m.title }.sort.join(", ") end - def evidence_sha - evidences.first&.summary_sha - end - - def evidence_summary - evidences.first&.summary || {} - end - private def actual_sha @@ -100,10 +90,6 @@ class Release < ApplicationRecord repository.find_tag(tag) end end - - def notify_new_release - NewReleaseWorker.perform_async(id) - end end Release.prepend_if_ee('EE::Release') diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index 0334d63dd36..8e7612e63c8 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -106,7 +106,23 @@ class RemoteMirror < ApplicationRecord update_status == 'started' end - def update_repository(options) + def update_repository + Gitlab::Git::RemoteMirror.new( + project.repository.raw, + remote_name, + **options_for_update + ).update + end + + def options_for_update + options = { + keep_divergent_refs: keep_divergent_refs? + } + + if only_protected_branches? + options[:only_branches_matching] = project.protected_branches.pluck(:name) + end + if ssh_mirror_url? if ssh_key_auth? && ssh_private_key.present? options[:ssh_key] = ssh_private_key @@ -117,13 +133,7 @@ class RemoteMirror < ApplicationRecord end end - options[:keep_divergent_refs] = keep_divergent_refs? - - Gitlab::Git::RemoteMirror.new( - project.repository.raw, - remote_name, - **options - ).update + options end def sync? diff --git a/app/models/repository.rb b/app/models/repository.rb index a9ef0504a3d..2673033ff1f 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1120,6 +1120,17 @@ class Repository end end + # TODO: pass this in directly to `Blob` rather than delegating it to here + # + # https://gitlab.com/gitlab-org/gitlab/-/issues/201886 + def lfs_enabled? + if container.is_a?(Project) + container.lfs_enabled? + else + false # LFS is not supported for snippet or group repositories + end + end + private # TODO Genericize finder, later split this on finders by Ref or Oid diff --git a/app/models/resource_label_event.rb b/app/models/resource_label_event.rb index cd47c154eef..845be408d5e 100644 --- a/app/models/resource_label_event.rb +++ b/app/models/resource_label_event.rb @@ -2,16 +2,14 @@ class ResourceLabelEvent < ResourceEvent include CacheMarkdownField + include IssueResourceEvent + include MergeRequestResourceEvent cache_markdown_field :reference - belongs_to :issue - belongs_to :merge_request belongs_to :label scope :inc_relations, -> { includes(:label, :user) } - scope :by_issue, ->(issue) { where(issue_id: issue.id) } - scope :by_merge_request, ->(merge_request) { where(merge_request_id: merge_request.id) } validates :label, presence: { unless: :importing? }, on: :create validate :exactly_one_issuable diff --git a/app/models/resource_milestone_event.rb b/app/models/resource_milestone_event.rb index a40af22061e..039f26d8e3f 100644 --- a/app/models/resource_milestone_event.rb +++ b/app/models/resource_milestone_event.rb @@ -2,14 +2,11 @@ class ResourceMilestoneEvent < ResourceEvent include IgnorableColumns + include IssueResourceEvent + include MergeRequestResourceEvent - belongs_to :issue - belongs_to :merge_request belongs_to :milestone - scope :by_issue, ->(issue) { where(issue_id: issue.id) } - scope :by_merge_request, ->(merge_request) { where(merge_request_id: merge_request.id) } - validate :exactly_one_issuable enum action: { @@ -25,4 +22,8 @@ class ResourceMilestoneEvent < ResourceEvent def self.issuable_attrs %i(issue merge_request).freeze end + + def milestone_title + milestone&.title + end end diff --git a/app/models/resource_state_event.rb b/app/models/resource_state_event.rb new file mode 100644 index 00000000000..1d6573b180f --- /dev/null +++ b/app/models/resource_state_event.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class ResourceStateEvent < ResourceEvent + include IssueResourceEvent + include MergeRequestResourceEvent + + validate :exactly_one_issuable + + # state is used for issue and merge request states. + enum state: Issue.available_states.merge(MergeRequest.available_states).merge(reopened: 5) + + def self.issuable_attrs + %i(issue merge_request).freeze + end +end diff --git a/app/models/resource_weight_event.rb b/app/models/resource_weight_event.rb index e0cc0c87a83..bbabd54325e 100644 --- a/app/models/resource_weight_event.rb +++ b/app/models/resource_weight_event.rb @@ -3,7 +3,5 @@ class ResourceWeightEvent < ResourceEvent validates :issue, presence: true - belongs_to :issue - - scope :by_issue, ->(issue) { where(issue_id: issue.id) } + include IssueResourceEvent end diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index f3a9293376f..4165d3b753f 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -76,12 +76,14 @@ class SentNotification < ApplicationRecord def position=(new_position) if new_position.is_a?(String) - new_position = JSON.parse(new_position) rescue nil + new_position = Gitlab::Json.parse(new_position) rescue nil end if new_position.is_a?(Hash) new_position = new_position.with_indifferent_access new_position = Gitlab::Diff::Position.new(new_position) + else + new_position = nil end super(new_position) diff --git a/app/models/service.rb b/app/models/service.rb index 543869c71d6..fb4d9a77077 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -12,7 +12,7 @@ class Service < ApplicationRecord alerts asana assembla bamboo bugzilla buildkite campfire custom_issue_tracker discord drone_ci emails_on_push external_wiki flowdock hangouts_chat hipchat irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email - pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit youtrack + pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack ].freeze DEV_SERVICE_NAMES = %w[ @@ -81,6 +81,10 @@ class Service < ApplicationRecord active end + def operating? + active && persisted? + end + def show_active_box? true end @@ -345,14 +349,6 @@ class Service < ApplicationRecord service end - def deprecated? - false - end - - def deprecation_message - nil - end - # override if needed def supports_data_fields? false diff --git a/app/models/snippet.rb b/app/models/snippet.rb index dbf600cf0df..72ebdf61787 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -15,9 +15,11 @@ class Snippet < ApplicationRecord include FromUnion include IgnorableColumns include HasRepository + include AfterCommitQueue extend ::Gitlab::Utils::Override - MAX_FILE_COUNT = 1 + MAX_FILE_COUNT = 10 + MAX_SINGLE_FILE_COUNT = 1 cache_markdown_field :title, pipeline: :single_line cache_markdown_field :description @@ -101,6 +103,10 @@ class Snippet < ApplicationRecord where(project_id: nil) end + def self.only_project_snippets + where.not(project_id: nil) + end + def self.only_include_projects_visible_to(current_user = nil) levels = Gitlab::VisibilityLevel.levels_for_user(current_user) @@ -164,6 +170,10 @@ class Snippet < ApplicationRecord Snippet.find_by(id: id, project: project) end + def self.max_file_limit(user) + Feature.enabled?(:snippet_multiple_files, user) ? MAX_FILE_COUNT : MAX_SINGLE_FILE_COUNT + end + def initialize(attributes = {}) # We can't use default_value_for because the database has a default # value of 0 for visibility_level. If someone attempts to create a @@ -199,7 +209,7 @@ class Snippet < ApplicationRecord def blobs return [] unless repository_exists? - repository.ls_files(repository.root_ref).map { |file| Blob.lazy(self, repository.root_ref, file) } + repository.ls_files(repository.root_ref).map { |file| Blob.lazy(repository, repository.root_ref, file) } end def hook_attrs @@ -318,8 +328,10 @@ class Snippet < ApplicationRecord Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}") end - def versioned_enabled_for?(user) - ::Feature.enabled?(:version_snippets, user) && repository_exists? + def file_name_on_repo + return if repository.empty? + + repository.ls_files(repository.root_ref).first end class << self @@ -334,17 +346,6 @@ class Snippet < ApplicationRecord fuzzy_search(query, [:title, :description, :file_name]) end - # Searches for snippets with matching content. - # - # This method uses ILIKE on PostgreSQL and LIKE on MySQL. - # - # query - The search query as a String. - # - # Returns an ActiveRecord::Relation. - def search_code(query) - fuzzy_search(query, [:content]) - end - def parent_class ::Project end diff --git a/app/models/snippet_repository.rb b/app/models/snippet_repository.rb index e60dbb4d141..2276851b7a1 100644 --- a/app/models/snippet_repository.rb +++ b/app/models/snippet_repository.rb @@ -7,6 +7,8 @@ class SnippetRepository < ApplicationRecord EMPTY_FILE_PATTERN = /^#{DEFAULT_EMPTY_FILE_NAME}(\d+)\.txt$/.freeze CommitError = Class.new(StandardError) + InvalidPathError = Class.new(CommitError) + InvalidSignatureError = Class.new(CommitError) belongs_to :snippet, inverse_of: :snippet_repository @@ -40,8 +42,12 @@ class SnippetRepository < ApplicationRecord rescue Gitlab::Git::Index::IndexError, Gitlab::Git::CommitError, Gitlab::Git::PreReceiveError, - Gitlab::Git::CommandError => e - raise CommitError, e.message + Gitlab::Git::CommandError, + ArgumentError => error + + logger.error(message: "Snippet git error. Reason: #{error.message}", snippet: snippet.id) + + raise commit_error_exception(error) end def transform_file_entries(files) @@ -85,4 +91,24 @@ class SnippetRepository < ApplicationRecord def build_empty_file_name(index) "#{DEFAULT_EMPTY_FILE_NAME}#{index}.txt" end + + def commit_error_exception(err) + if invalid_path_error?(err) + InvalidPathError.new('Invalid file name') # To avoid returning the message with the path included + elsif invalid_signature_error?(err) + InvalidSignatureError.new(err.message) + else + CommitError.new(err.message) + end + end + + def invalid_path_error?(err) + err.is_a?(Gitlab::Git::Index::IndexError) && + err.message.downcase.start_with?('invalid path', 'path cannot include directory traversal') + end + + def invalid_signature_error?(err) + err.is_a?(ArgumentError) && + err.message.downcase.match?(/failed to parse signature/) + end end diff --git a/app/models/ssh_host_key.rb b/app/models/ssh_host_key.rb index 9bd35d30845..72690ad7d04 100644 --- a/app/models/ssh_host_key.rb +++ b/app/models/ssh_host_key.rb @@ -24,6 +24,7 @@ class SshHostKey # This is achieved by making the lifetime shorter than the refresh interval. self.reactive_cache_refresh_interval = 15.minutes self.reactive_cache_lifetime = 10.minutes + self.reactive_cache_work_type = :external_dependency def self.find_by(opts = {}) opts = HashWithIndifferentAccess.new(opts) diff --git a/app/models/state_note.rb b/app/models/state_note.rb new file mode 100644 index 00000000000..cbcb1c2b49d --- /dev/null +++ b/app/models/state_note.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class StateNote < SyntheticNote + def self.from_event(event, resource: nil, resource_parent: nil) + attrs = note_attributes(event.state, event, resource, resource_parent) + + StateNote.new(attrs) + end + + def note_html + @note_html ||= "<p dir=\"auto\">#{note_text(html: true)}</p>" + end + + private + + def note_text(html: false) + event.state + end +end diff --git a/app/models/storage/hashed.rb b/app/models/storage/hashed.rb index 3dea50ab98b..c61cd3b6b30 100644 --- a/app/models/storage/hashed.rb +++ b/app/models/storage/hashed.rb @@ -6,6 +6,7 @@ module Storage delegate :gitlab_shell, :repository_storage, to: :container REPOSITORY_PATH_PREFIX = '@hashed' + GROUP_REPOSITORY_PATH_PREFIX = '@groups' SNIPPET_REPOSITORY_PATH_PREFIX = '@snippets' POOL_PATH_PREFIX = '@pools' diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index b881a43ad4d..4e14bb4e92c 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -15,6 +15,7 @@ class SystemNoteMetadata < ApplicationRecord ICON_TYPES = %w[ commit description merge confidential visible label assignee cross_reference + designs_added designs_modified designs_removed designs_discussion_added title time_tracking branch milestone discussion task moved opened closed merged duplicate locked unlocked outdated tag due_date pinned_embed cherry_pick health_status diff --git a/app/models/timelog.rb b/app/models/timelog.rb index f52dd74d4c9..c0aac6f27aa 100644 --- a/app/models/timelog.rb +++ b/app/models/timelog.rb @@ -16,8 +16,8 @@ class Timelog < ApplicationRecord ) end - scope :between_dates, -> (start_date, end_date) do - where('spent_at BETWEEN ? AND ?', start_date, end_date) + scope :between_times, -> (start_time, end_time) do + where('spent_at BETWEEN ? AND ?', start_time, end_time) end def issuable diff --git a/app/models/todo.rb b/app/models/todo.rb index d337ef33051..dc42551f0ab 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -110,7 +110,7 @@ class Todo < ApplicationRecord base = where.not(state: new_state).except(:order) ids = base.pluck(:id) - base.update_all(state: new_state) + base.update_all(state: new_state, updated_at: Time.now) ids end @@ -183,6 +183,10 @@ class Todo < ApplicationRecord target_type == "Commit" end + def for_design? + target_type == DesignManagement::Design.name + end + # override to return commits, which are not active record def target if for_commit? diff --git a/app/models/user.rb b/app/models/user.rb index 1b087da3a2f..b2d3978551e 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -24,6 +24,7 @@ class User < ApplicationRecord include HasUniqueInternalUsers include IgnorableColumns include UpdateHighestRole + include HasUserType DEFAULT_NOTIFICATION_LEVEL = :participating @@ -57,6 +58,10 @@ class User < ApplicationRecord devise :lockable, :recoverable, :rememberable, :trackable, :validatable, :omniauthable, :confirmable, :registerable + # This module adds async behaviour to Devise emails + # and should be added after Devise modules are initialized. + include AsyncDeviseEmail + BLOCKED_MESSAGE = "Your account has been blocked. Please contact your GitLab " \ "administrator if you think this is an error." LOGIN_FORBIDDEN = "Your account does not have the required permission to login. Please contact your GitLab " \ @@ -64,9 +69,8 @@ class User < ApplicationRecord MINIMUM_INACTIVE_DAYS = 180 - enum user_type: ::UserTypeEnums.types - - ignore_column :bot_type, remove_with: '12.11', remove_after: '2020-04-22' + ignore_column :bot_type, remove_with: '13.1', remove_after: '2020-05-22' + ignore_column :ghost, remove_with: '13.2', remove_after: '2020-06-22' # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour @@ -88,6 +92,9 @@ class User < ApplicationRecord # Virtual attribute for authenticating by either username or email attr_accessor :login + # Virtual attribute for impersonator + attr_accessor :impersonator + # # Relations # @@ -166,6 +173,8 @@ class User < ApplicationRecord has_many :term_agreements belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' + has_many :metrics_users_starred_dashboards, class_name: 'Metrics::UsersStarredDashboard', inverse_of: :user + has_one :status, class_name: 'UserStatus' has_one :user_preference has_one :user_detail @@ -246,15 +255,12 @@ class User < ApplicationRecord enum layout: { fixed: 0, fluid: 1 } # User's Dashboard preference - # Note: When adding an option, it MUST go on the end of the array. enum dashboard: { projects: 0, stars: 1, project_activity: 2, starred_project_activity: 3, groups: 4, todos: 5, issues: 6, merge_requests: 7, operations: 8 } # User's Project preference - # Note: When adding an option, it MUST go on the end of the array. enum project_view: { readme: 0, activity: 1, files: 2 } # User's role - # Note: When adding an option, it MUST go on the end of the array. enum role: { software_developer: 0, development_team_lead: 1, devops_engineer: 2, systems_administrator: 3, security_analyst: 4, data_analyst: 5, product_manager: 6, product_designer: 7, other: 8 }, _suffix: true delegate :path, to: :namespace, allow_nil: true, prefix: true @@ -321,32 +327,26 @@ class User < ApplicationRecord scope :admins, -> { where(admin: true) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) } scope :external, -> { where(external: true) } + scope :confirmed, -> { where.not(confirmed_at: nil) } scope :active, -> { with_state(:active).non_internal } scope :active_without_ghosts, -> { with_state(:active).without_ghosts } - scope :without_ghosts, -> { where('ghost IS NOT TRUE') } scope :deactivated, -> { with_state(:deactivated).non_internal } scope :without_projects, -> { joins('LEFT JOIN project_authorizations ON users.id = project_authorizations.user_id').where(project_authorizations: { user_id: nil }) } - scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) } - scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) } - scope :order_recent_last_activity, -> { reorder(Gitlab::Database.nulls_last_order('last_activity_on', 'DESC')) } - scope :order_oldest_last_activity, -> { reorder(Gitlab::Database.nulls_first_order('last_activity_on', 'ASC')) } - scope :confirmed, -> { where.not(confirmed_at: nil) } scope :by_username, -> (usernames) { iwhere(username: Array(usernames).map(&:to_s)) } scope :for_todos, -> (todos) { where(id: todos.select(:user_id)) } scope :with_emails, -> { preload(:emails) } scope :with_dashboard, -> (dashboard) { where(dashboard: dashboard) } scope :with_public_profile, -> { where(private_profile: false) } - scope :bots, -> { where(user_type: UserTypeEnums.bots.values) } - scope :bots_without_project_bot, -> { bots.where.not(user_type: UserTypeEnums.bots[:project_bot]) } - scope :with_project_bots, -> { humans.or(where.not(user_type: UserTypeEnums.bots.except(:project_bot).values)) } - scope :humans, -> { where(user_type: nil) } - scope :with_expiring_and_not_notified_personal_access_tokens, ->(at) do where('EXISTS (?)', ::PersonalAccessToken .where('personal_access_tokens.user_id = users.id') .expiring_and_not_notified(at).select(1)) end + scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) } + scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) } + scope :order_recent_last_activity, -> { reorder(Gitlab::Database.nulls_last_order('last_activity_on', 'DESC')) } + scope :order_oldest_last_activity, -> { reorder(Gitlab::Database.nulls_first_order('last_activity_on', 'ASC')) } def active_for_authentication? super && can?(:log_in) @@ -624,7 +624,7 @@ class User < ApplicationRecord # owns records previously belonging to deleted users. def ghost email = 'ghost%s@example.com' - unique_internal(where(ghost: true, user_type: :ghost), 'ghost', email) do |u| + unique_internal(where(user_type: :ghost), 'ghost', email) do |u| u.bio = _('This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.') u.name = 'Ghost User' end @@ -639,6 +639,16 @@ class User < ApplicationRecord end end + def migration_bot + email_pattern = "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}" + + unique_internal(where(user_type: :migration_bot), 'migration-bot', email_pattern) do |u| + u.bio = 'The GitLab migration bot' + u.name = 'GitLab Migration Bot' + u.confirmed_at = Time.zone.now + end + end + # Return true if there is only single non-internal user in the deployment, # ghost user is ignored. def single_user? @@ -650,43 +660,14 @@ class User < ApplicationRecord end end - def full_path - username - end - - def bot? - UserTypeEnums.bots.has_key?(user_type) - end - - # The explicit check for project_bot will be removed with Bot Categorization - # Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/213945 - def internal? - ghost? || (bot? && !project_bot?) - end - - # We are transitioning from ghost boolean column to user_type - # so we need to read from old column for now - # @see https://gitlab.com/gitlab-org/gitlab/-/issues/210025 - def ghost? - ghost - end - - # The explicit check for project_bot will be removed with Bot Categorization - # Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/213945 - def self.internal - where(ghost: true).or(bots_without_project_bot) - end - - # The explicit check for project_bot will be removed with Bot Categorization - # Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/213945 - def self.non_internal - without_ghosts.with_project_bots - end - # # Instance methods # + def full_path + username + end + def to_param username end @@ -1700,16 +1681,6 @@ class User < ApplicationRecord callouts.any? end - def gitlab_employee? - strong_memoize(:gitlab_employee) do - if Feature.enabled?(:gitlab_employee_badge) && Gitlab.com? - Mail::Address.new(email).domain == "gitlab.com" && confirmed? - else - false - end - end - end - # Load the current highest access by looking directly at the user's memberships def current_highest_access_level members.non_request.maximum(:access_level) @@ -1719,8 +1690,8 @@ class User < ApplicationRecord !confirmed? && !confirmation_period_valid? end - def organization - gitlab_employee? ? 'GitLab' : super + def impersonated? + impersonator.present? end protected @@ -1779,13 +1750,6 @@ class User < ApplicationRecord ApplicationSetting.current_without_cache&.usage_stats_set_by_user_id == self.id end - # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration - def send_devise_notification(notification, *args) - return true unless can?(:receive_notifications) - - devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend - end - def ensure_user_rights_and_limits if external? self.can_create_group = false @@ -1834,7 +1798,6 @@ class User < ApplicationRecord end def check_email_restrictions - return unless Feature.enabled?(:email_restrictions) return unless Gitlab::CurrentSettings.email_restrictions_enabled? restrictions = Gitlab::CurrentSettings.email_restrictions diff --git a/app/models/user_type_enums.rb b/app/models/user_type_enums.rb deleted file mode 100644 index cb5aac89ed3..00000000000 --- a/app/models/user_type_enums.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module UserTypeEnums - def self.types - @types ||= bots.merge(human: nil, ghost: 5) - end - - def self.bots - @bots ||= { alert_bot: 2, project_bot: 6 }.with_indifferent_access - end -end - -UserTypeEnums.prepend_if_ee('EE::UserTypeEnums') diff --git a/app/models/wiki.rb b/app/models/wiki.rb new file mode 100644 index 00000000000..54bcec32095 --- /dev/null +++ b/app/models/wiki.rb @@ -0,0 +1,233 @@ +# frozen_string_literal: true + +class Wiki + extend ::Gitlab::Utils::Override + include HasRepository + include Gitlab::Utils::StrongMemoize + + MARKUPS = { # rubocop:disable Style/MultilineIfModifier + 'Markdown' => :markdown, + 'RDoc' => :rdoc, + 'AsciiDoc' => :asciidoc, + 'Org' => :org + }.freeze unless defined?(MARKUPS) + + CouldNotCreateWikiError = Class.new(StandardError) + + HOMEPAGE = 'home' + SIDEBAR = '_sidebar' + + TITLE_ORDER = 'title' + CREATED_AT_ORDER = 'created_at' + DIRECTION_DESC = 'desc' + DIRECTION_ASC = 'asc' + + attr_reader :container, :user + + # Returns a string describing what went wrong after + # an operation fails. + attr_reader :error_message + + def self.for_container(container, user = nil) + "#{container.class.name}Wiki".constantize.new(container, user) + end + + def initialize(container, user = nil) + @container = container + @user = user + end + + def path + container.path + '.wiki' + end + + # Returns the Gitlab::Git::Wiki object. + def wiki + strong_memoize(:wiki) do + create_wiki_repository + Gitlab::Git::Wiki.new(repository.raw) + end + end + + def create_wiki_repository + repository.create_if_not_exists + + raise CouldNotCreateWikiError unless repository_exists? + rescue => err + Gitlab::ErrorTracking.track_exception(err, wiki: { + container_type: container.class.name, + container_id: container.id, + full_path: full_path, + disk_path: disk_path + }) + + raise CouldNotCreateWikiError + end + + def has_home_page? + !!find_page(HOMEPAGE) + end + + def empty? + list_pages(limit: 1).empty? + end + + def exists? + !empty? + end + + # Lists wiki pages of the repository. + # + # limit - max number of pages returned by the method. + # sort - criterion by which the pages are sorted. + # direction - order of the sorted pages. + # load_content - option, which specifies whether the content inside the page + # will be loaded. + # + # Returns an Array of GitLab WikiPage instances or an + # empty Array if this Wiki has no pages. + def list_pages(limit: 0, sort: nil, direction: DIRECTION_ASC, load_content: false) + wiki.list_pages( + limit: limit, + sort: sort, + direction_desc: direction == DIRECTION_DESC, + load_content: load_content + ).map do |page| + WikiPage.new(self, page) + end + end + + def sidebar_entries(limit: Gitlab::WikiPages::MAX_SIDEBAR_PAGES, **options) + pages = list_pages(**options.merge(limit: limit + 1)) + limited = pages.size > limit + pages = pages.first(limit) if limited + + [WikiPage.group_by_directory(pages), limited] + end + + # Finds a page within the repository based on a tile + # or slug. + # + # title - The human readable or parameterized title of + # the page. + # + # Returns an initialized WikiPage instance or nil + def find_page(title, version = nil) + page_title, page_dir = page_title_and_dir(title) + + if page = wiki.page(title: page_title, version: version, dir: page_dir) + WikiPage.new(self, page) + end + end + + def find_sidebar(version = nil) + find_page(SIDEBAR, version) + end + + def find_file(name, version = nil) + wiki.file(name, version) + end + + def create_page(title, content, format = :markdown, message = nil) + commit = commit_details(:created, message, title) + + wiki.write_page(title, format.to_sym, content, commit) + + update_container_activity + rescue Gitlab::Git::Wiki::DuplicatePageError => e + @error_message = "Duplicate page: #{e.message}" + false + end + + def update_page(page, content:, title: nil, format: :markdown, message: nil) + commit = commit_details(:updated, message, page.title) + + wiki.update_page(page.path, title || page.name, format.to_sym, content, commit) + + update_container_activity + end + + def delete_page(page, message = nil) + return unless page + + wiki.delete_page(page.path, commit_details(:deleted, message, page.title)) + + update_container_activity + end + + def page_title_and_dir(title) + return unless title + + title_array = title.split("/") + title = title_array.pop + [title, title_array.join("/")] + end + + def ensure_repository + raise CouldNotCreateWikiError unless wiki.repository_exists? + end + + def hook_attrs + { + web_url: web_url, + git_ssh_url: ssh_url_to_repo, + git_http_url: http_url_to_repo, + path_with_namespace: full_path, + default_branch: default_branch + } + end + + override :repository + def repository + @repository ||= Repository.new(full_path, container, shard: repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::WIKI) + end + + def repository_storage + raise NotImplementedError + end + + def hashed_storage? + raise NotImplementedError + end + + override :full_path + def full_path + container.full_path + '.wiki' + end + alias_method :id, :full_path + + # @deprecated use full_path when you need it for an URL route or disk_path when you want to point to the filesystem + alias_method :path_with_namespace, :full_path + + override :default_branch + def default_branch + wiki.class.default_ref + end + + def wiki_base_path + Gitlab.config.gitlab.relative_url_root + web_url(only_path: true).sub(%r{/#{Wiki::HOMEPAGE}\z}, '') + end + + private + + def commit_details(action, message = nil, title = nil) + commit_message = message.presence || default_message(action, title) + git_user = Gitlab::Git::User.from_gitlab(user) + + Gitlab::Git::Wiki::CommitDetails.new(user.id, + git_user.username, + git_user.name, + git_user.email, + commit_message) + end + + def default_message(action, title) + "#{user.username} #{action} page: #{title}" + end + + def update_container_activity + container.after_wiki_activity + end +end + +Wiki.prepend_if_ee('EE::Wiki') diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 9c887fc87f3..319cdd38d93 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -26,7 +26,7 @@ class WikiPage def eql?(other) return false unless other.present? && other.is_a?(self.class) - slug == other.slug && wiki.project == other.wiki.project + slug == other.slug && wiki.container == other.wiki.container end alias_method :==, :eql? @@ -66,9 +66,9 @@ class WikiPage validates :content, presence: true validate :validate_path_limits, if: :title_changed? - # The GitLab ProjectWiki instance. + # The GitLab Wiki instance. attr_reader :wiki - delegate :project, to: :wiki + delegate :container, to: :wiki # The raw Gitlab::Git::WikiPage instance. attr_reader :page @@ -83,7 +83,7 @@ class WikiPage # Construct a new WikiPage # - # @param [ProjectWiki] wiki + # @param [Wiki] wiki # @param [Gitlab::Git::WikiPage] page def initialize(wiki, page = nil) @wiki = wiki @@ -95,29 +95,29 @@ class WikiPage # The escaped URL path of this page. def slug - @attributes[:slug].presence || wiki.wiki.preview_slug(title, format) + attributes[:slug].presence || wiki.wiki.preview_slug(title, format) end alias_method :to_param, :slug def human_title - return 'Home' if title == 'home' + return 'Home' if title == Wiki::HOMEPAGE title end # The formatted title of this page. def title - @attributes[:title] || '' + attributes[:title] || '' end # Sets the title of this page. def title=(new_title) - @attributes[:title] = new_title + attributes[:title] = new_title end def raw_content - @attributes[:content] ||= @page&.text_data + attributes[:content] ||= page&.text_data end # The hierarchy of the directory this page is contained in. @@ -127,7 +127,7 @@ class WikiPage # The markup format for the page. def format - @attributes[:format] || :markdown + attributes[:format] || :markdown end # The commit message for this page version. @@ -151,13 +151,13 @@ class WikiPage def versions(options = {}) return [] unless persisted? - wiki.wiki.page_versions(@page.path, options) + wiki.wiki.page_versions(page.path, options) end def count_versions return [] unless persisted? - wiki.wiki.count_page_versions(@page.path) + wiki.wiki.count_page_versions(page.path) end def last_version @@ -173,7 +173,7 @@ class WikiPage def historical? return false unless last_commit_sha && version - @page.historical? && last_commit_sha != version.sha + page.historical? && last_commit_sha != version.sha end # Returns boolean True or False if this instance @@ -185,7 +185,7 @@ class WikiPage # Returns boolean True or False if this instance # has been fully created on disk or not. def persisted? - @page.present? + page.present? end # Creates a new Wiki Page. @@ -195,7 +195,7 @@ class WikiPage # :content - The raw markup content. # :format - Optional symbol representing the # content format. Can be any type - # listed in the ProjectWiki::MARKUPS + # listed in the Wiki::MARKUPS # Hash. # :message - Optional commit message to set on # the new page. @@ -215,7 +215,7 @@ class WikiPage # attrs - Hash of attributes to be updated on the page. # :content - The raw markup content to replace the existing. # :format - Optional symbol representing the content format. - # See ProjectWiki::MARKUPS Hash for available formats. + # See Wiki::MARKUPS Hash for available formats. # :message - Optional commit message to set on the new version. # :last_commit_sha - Optional last commit sha to validate the page unchanged. # :title - The Title (optionally including dir) to replace existing title @@ -232,13 +232,13 @@ class WikiPage update_attributes(attrs) if title.present? && title_changed? && wiki.find_page(title).present? - @attributes[:title] = @page.title + attributes[:title] = page.title raise PageRenameError end save do wiki.update_page( - @page, + page, content: raw_content, format: format, message: attrs[:message], @@ -251,7 +251,7 @@ class WikiPage # # Returns boolean True or False. def delete - if wiki.delete_page(@page) + if wiki.delete_page(page) true else false @@ -261,6 +261,7 @@ class WikiPage # Relative path to the partial to be used when rendering collections # of this object. def to_partial_path + # TODO: Move into shared/ with https://gitlab.com/gitlab-org/gitlab/-/issues/196054 'projects/wikis/wiki_page' end @@ -270,7 +271,7 @@ class WikiPage def title_changed? if persisted? - old_title, old_dir = wiki.page_title_and_dir(self.class.unhyphenize(@page.url_path)) + old_title, old_dir = wiki.page_title_and_dir(self.class.unhyphenize(page.url_path)) new_title, new_dir = wiki.page_title_and_dir(self.class.unhyphenize(title)) new_title != old_title || (title.include?('/') && new_dir != old_dir) @@ -287,13 +288,17 @@ class WikiPage attrs.slice!(:content, :format, :message, :title) clear_memoization(:parsed_content) if attrs.has_key?(:content) - @attributes.merge!(attrs) + attributes.merge!(attrs) end def to_ability_name 'wiki_page' end + def version_commit_timestamp + version&.commit&.committed_date + end + private def serialize_front_matter(hash) @@ -303,7 +308,7 @@ class WikiPage end def update_front_matter(attrs) - return unless Gitlab::WikiPages::FrontMatterParser.enabled?(project) + return unless Gitlab::WikiPages::FrontMatterParser.enabled?(container) return unless attrs.has_key?(:front_matter) fm_yaml = serialize_front_matter(attrs[:front_matter]) @@ -314,7 +319,7 @@ class WikiPage def parsed_content strong_memoize(:parsed_content) do - Gitlab::WikiPages::FrontMatterParser.new(raw_content, project).parse + Gitlab::WikiPages::FrontMatterParser.new(raw_content, container).parse end end @@ -325,7 +330,7 @@ class WikiPage title = deep_title_squish(title) current_dirname = File.dirname(title) - if @page.present? + if persisted? return title[1..-1] if current_dirname == '/' return File.join([directory.presence, title].compact) if current_dirname == '.' end @@ -362,9 +367,11 @@ class WikiPage end def validate_path_limits - *dirnames, title = @attributes[:title].split('/') + return unless title.present? + + *dirnames, filename = title.split('/') - if title && title.bytesize > Gitlab::WikiPages::MAX_TITLE_BYTES + if filename && filename.bytesize > Gitlab::WikiPages::MAX_TITLE_BYTES errors.add(:title, _("exceeds the limit of %{bytes} bytes") % { bytes: Gitlab::WikiPages::MAX_TITLE_BYTES }) diff --git a/app/models/wiki_page/meta.rb b/app/models/wiki_page/meta.rb index 2af7d86ebcc..474968122b1 100644 --- a/app/models/wiki_page/meta.rb +++ b/app/models/wiki_page/meta.rb @@ -5,6 +5,7 @@ class WikiPage include Gitlab::Utils::StrongMemoize CanonicalSlugConflictError = Class.new(ActiveRecord::RecordInvalid) + WikiPageInvalid = Class.new(ArgumentError) self.table_name = 'wiki_page_meta' @@ -23,46 +24,62 @@ class WikiPage alias_method :resource_parent, :project - # Return the (updated) WikiPage::Meta record for a given wiki page - # - # If none is found, then a new record is created, and its fields are set - # to reflect the wiki_page passed. - # - # @param [String] last_known_slug - # @param [WikiPage] wiki_page - # - # As with all `find_or_create` methods, this one raises errors on - # validation issues. - def self.find_or_create(last_known_slug, wiki_page) - project = wiki_page.wiki.project - known_slugs = [last_known_slug, wiki_page.slug].compact.uniq - raise 'no slugs!' if known_slugs.empty? - - transaction do - found = find_by_canonical_slug(known_slugs, project) - meta = found || create(title: wiki_page.title, project_id: project.id) - - meta.update_state(found.nil?, known_slugs, wiki_page) - - # We don't need to run validations here, since find_by_canonical_slug - # guarantees that there is no conflict in canonical_slug, and DB - # constraints on title and project_id enforce our other invariants - # This saves us a query. - meta + class << self + # Return the (updated) WikiPage::Meta record for a given wiki page + # + # If none is found, then a new record is created, and its fields are set + # to reflect the wiki_page passed. + # + # @param [String] last_known_slug + # @param [WikiPage] wiki_page + # + # This method raises errors on validation issues. + def find_or_create(last_known_slug, wiki_page) + raise WikiPageInvalid unless wiki_page.valid? + + project = wiki_page.wiki.project + known_slugs = [last_known_slug, wiki_page.slug].compact.uniq + raise 'No slugs found! This should not be possible.' if known_slugs.empty? + + transaction do + updates = wiki_page_updates(wiki_page) + found = find_by_canonical_slug(known_slugs, project) + meta = found || create!(updates.merge(project_id: project.id)) + + meta.update_state(found.nil?, known_slugs, wiki_page, updates) + + # We don't need to run validations here, since find_by_canonical_slug + # guarantees that there is no conflict in canonical_slug, and DB + # constraints on title and project_id enforce our other invariants + # This saves us a query. + meta + end end - end - def self.find_by_canonical_slug(canonical_slug, project) - meta, conflict = with_canonical_slug(canonical_slug) - .where(project_id: project.id) - .limit(2) + def find_by_canonical_slug(canonical_slug, project) + meta, conflict = with_canonical_slug(canonical_slug) + .where(project_id: project.id) + .limit(2) - if conflict.present? - meta.errors.add(:canonical_slug, 'Duplicate value found') - raise CanonicalSlugConflictError.new(meta) + if conflict.present? + meta.errors.add(:canonical_slug, 'Duplicate value found') + raise CanonicalSlugConflictError.new(meta) + end + + meta end - meta + private + + def wiki_page_updates(wiki_page) + last_commit_date = wiki_page.version_commit_timestamp || Time.now.utc + + { + title: wiki_page.title, + created_at: last_commit_date, + updated_at: last_commit_date + } + end end def canonical_slug @@ -85,24 +102,21 @@ class WikiPage @canonical_slug = slug end - def update_state(created, known_slugs, wiki_page) - update_wiki_page_attributes(wiki_page) + def update_state(created, known_slugs, wiki_page, updates) + update_wiki_page_attributes(updates) insert_slugs(known_slugs, created, wiki_page.slug) self.canonical_slug = wiki_page.slug end - def update_columns(attrs = {}) - super(attrs.reverse_merge(updated_at: Time.now.utc)) - end - - def self.update_all(attrs = {}) - super(attrs.reverse_merge(updated_at: Time.now.utc)) - end - private - def update_wiki_page_attributes(page) - update_columns(title: page.title) unless page.title == title + def update_wiki_page_attributes(updates) + # Remove all unnecessary updates: + updates.delete(:updated_at) if updated_at == updates[:updated_at] + updates.delete(:created_at) if created_at <= updates[:created_at] + updates.delete(:title) if title == updates[:title] + + update_columns(updates) unless updates.empty? end def insert_slugs(strings, is_new, canonical_slug) diff --git a/app/models/x509_certificate.rb b/app/models/x509_certificate.rb index 75b711eab5b..428fd336a32 100644 --- a/app/models/x509_certificate.rb +++ b/app/models/x509_certificate.rb @@ -26,6 +26,8 @@ class X509Certificate < ApplicationRecord validates :x509_issuer_id, presence: true + scope :by_x509_issuer, ->(issuer) { where(x509_issuer_id: issuer.id) } + after_commit :mark_commit_signatures_unverified def self.safe_create!(attributes) @@ -33,6 +35,10 @@ class X509Certificate < ApplicationRecord .safe_find_or_create_by!(subject_key_identifier: attributes[:subject_key_identifier]) end + def self.serial_numbers(issuer) + by_x509_issuer(issuer).pluck(:serial_number) + end + def mark_commit_signatures_unverified X509CertificateRevokeWorker.perform_async(self.id) if revoked? end diff --git a/app/models/x509_commit_signature.rb b/app/models/x509_commit_signature.rb index ed7c638cecc..57d809f7cfb 100644 --- a/app/models/x509_commit_signature.rb +++ b/app/models/x509_commit_signature.rb @@ -41,4 +41,8 @@ class X509CommitSignature < ApplicationRecord Gitlab::X509::Commit.new(commit) end + + def user + commit.committer + end end |