diff options
Diffstat (limited to 'app/models')
83 files changed, 1611 insertions, 563 deletions
diff --git a/app/models/appearance.rb b/app/models/appearance.rb index f9c48482be7..ff15689ecac 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -8,7 +8,27 @@ class Appearance < ActiveRecord::Base validates :logo, file_size: { maximum: 1.megabyte } validates :header_logo, file_size: { maximum: 1.megabyte } + validate :single_appearance_row, on: :create + mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + + CACHE_KEY = 'current_appearance'.freeze + + after_commit :flush_redis_cache + + def self.current + Rails.cache.fetch(CACHE_KEY) { first } + end + + def flush_redis_cache + Rails.cache.delete(CACHE_KEY) + end + + def single_appearance_row + if self.class.any? + errors.add(:single_appearance_row, 'Only 1 appearances row can exist') + end + end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index bd7c4cd45ea..3568e72e463 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -13,6 +13,11 @@ class ApplicationSetting < ActiveRecord::Base [\r\n] # any number of newline characters }x + # Setting a key restriction to `-1` means that all keys of this type are + # forbidden. + FORBIDDEN_KEY_VALUE = KeyRestrictionValidator::FORBIDDEN + SUPPORTED_KEY_TYPES = %i[rsa dsa ecdsa ed25519].freeze + serialize :restricted_visibility_levels # rubocop:disable Cop/ActiveRecordSerialize serialize :import_sources # rubocop:disable Cop/ActiveRecordSerialize serialize :disabled_oauth_sign_in_sources, Array # rubocop:disable Cop/ActiveRecordSerialize @@ -146,6 +151,12 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { greater_than_or_equal_to: 0 } + SUPPORTED_KEY_TYPES.each do |type| + validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } + end + + validates :allowed_key_types, presence: true + validates_each :restricted_visibility_levels do |record, attr, value| value&.each do |level| unless Gitlab::VisibilityLevel.options.value?(level) @@ -171,6 +182,7 @@ class ApplicationSetting < ActiveRecord::Base end before_validation :ensure_uuid! + before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token @@ -221,6 +233,9 @@ class ApplicationSetting < ActiveRecord::Base default_group_visibility: Settings.gitlab.default_projects_features['visibility_level'], disabled_oauth_sign_in_sources: [], domain_whitelist: Settings.gitlab['domain_whitelist'], + dsa_key_restriction: 0, + ecdsa_key_restriction: 0, + ed25519_key_restriction: 0, gravatar_enabled: Settings.gravatar['enabled'], help_page_text: nil, help_page_hide_commercial_content: false, @@ -239,8 +254,10 @@ class ApplicationSetting < ActiveRecord::Base max_attachment_size: Settings.gitlab['max_attachment_size'], password_authentication_enabled: Settings.gitlab['password_authentication_enabled'], performance_bar_allowed_group_id: nil, + rsa_key_restriction: 0, plantuml_enabled: false, plantuml_url: nil, + project_export_enabled: true, recaptcha_enabled: false, repository_checks_enabled: true, repository_storages: ['default'], @@ -412,6 +429,18 @@ class ApplicationSetting < ActiveRecord::Base usage_ping_can_be_configured? && super end + def allowed_key_types + SUPPORTED_KEY_TYPES.select do |type| + key_restriction_for(type) != FORBIDDEN_KEY_VALUE + end + end + + def key_restriction_for(type) + attr_name = "#{type}_key_restriction" + + has_attribute?(attr_name) ? public_send(attr_name) : FORBIDDEN_KEY_VALUE # rubocop:disable GitlabSecurity/PublicSend + end + private def ensure_uuid! diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index 91b62dabbcd..4d1a15c53aa 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -17,6 +17,9 @@ class AwardEmoji < ActiveRecord::Base scope :downvotes, -> { where(name: DOWNVOTE_NAME) } scope :upvotes, -> { where(name: UPVOTE_NAME) } + after_save :expire_etag_cache + after_destroy :expire_etag_cache + class << self def votes_for_collection(ids, type) select('name', 'awardable_id', 'COUNT(*) as count') @@ -32,4 +35,8 @@ class AwardEmoji < ActiveRecord::Base def upvote? self.name == UPVOTE_NAME end + + def expire_etag_cache + awardable.try(:expire_etag_cache) + end end diff --git a/app/models/blob_viewer/composer_json.rb b/app/models/blob_viewer/composer_json.rb index ef8b4aef8e8..def4879fbb5 100644 --- a/app/models/blob_viewer/composer_json.rb +++ b/app/models/blob_viewer/composer_json.rb @@ -9,7 +9,7 @@ module BlobViewer end def manager_url - 'https://getcomposer.com/' + 'https://getcomposer.org/' end def package_name diff --git a/app/models/blob_viewer/notebook.rb b/app/models/blob_viewer/notebook.rb index 8632b8a9885..e00b47e6c17 100644 --- a/app/models/blob_viewer/notebook.rb +++ b/app/models/blob_viewer/notebook.rb @@ -2,7 +2,7 @@ module BlobViewer class Notebook < Base include Rich include ClientSide - + self.partial_name = 'notebook' self.extensions = %w(ipynb) self.binary = false diff --git a/app/models/board.rb b/app/models/board.rb index 97d0f550925..5bb7d3d3722 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -3,7 +3,19 @@ class Board < ActiveRecord::Base has_many :lists, -> { order(:list_type, :position) }, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - validates :project, presence: true + validates :project, presence: true, if: :project_needed? + + def project_needed? + true + end + + def parent + project + end + + def group_board? + false + end def backlog_list lists.merge(List.backlog).take diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 944725d91c3..0b561203914 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -14,10 +14,26 @@ class BroadcastMessage < ActiveRecord::Base default_value_for :color, '#E75E40' default_value_for :font, '#FFFFFF' + CACHE_KEY = 'broadcast_message_current'.freeze + + after_commit :flush_redis_cache + def self.current - Rails.cache.fetch("broadcast_message_current", expires_in: 1.minute) do - where('ends_at > :now AND starts_at <= :now', now: Time.zone.now).order([:created_at, :id]).to_a - end + messages = Rails.cache.fetch(CACHE_KEY) { current_and_future_messages.to_a } + + return messages if messages.empty? + + now_or_future = messages.select(&:now_or_future?) + + # If there are cached entries but none are to be displayed we'll purge the + # cache so we don't keep running this code all the time. + Rails.cache.delete(CACHE_KEY) if now_or_future.empty? + + now_or_future.select(&:now?) + end + + def self.current_and_future_messages + where('ends_at > :now', now: Time.zone.now).order_id_asc end def active? @@ -31,4 +47,20 @@ class BroadcastMessage < ActiveRecord::Base def ended? ends_at < Time.zone.now end + + def now? + (starts_at..ends_at).cover?(Time.zone.now) + end + + def future? + starts_at > Time.zone.now + end + + def now_or_future? + now? || future? + end + + def flush_redis_cache + Rails.cache.delete(CACHE_KEY) + end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 569a71d36c1..c41355f5aff 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -3,6 +3,7 @@ module Ci include TokenAuthenticatable include AfterCommitQueue include Presentable + include Importable belongs_to :runner belongs_to :trigger_request @@ -34,6 +35,7 @@ module Ci scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } + scope :ref_protected, -> { where(protected: true) } mount_uploader :artifacts_file, ArtifactUploader mount_uploader :artifacts_metadata, ArtifactUploader @@ -46,7 +48,10 @@ module Ci before_save :ensure_token before_destroy { unscoped_project } - after_create :execute_hooks + after_create do |build| + run_after_commit { BuildHooksWorker.perform_async(build.id) } + end + after_commit :update_project_statistics_after_save, on: [:create, :update] after_commit :update_project_statistics, on: :destroy @@ -194,10 +199,7 @@ module Ci # * Maximum length is 63 bytes # * First/Last Character is not a hyphen def ref_slug - ref.to_s - .downcase - .gsub(/[^a-z0-9]/, '-')[0..62] - .gsub(/(\A-+|-+\z)/, '') + Gitlab::Utils.slugify(ref.to_s) end # Variables whose value does not depend on environment @@ -214,6 +216,7 @@ module Ci variables += runner.predefined_variables if runner variables += project.container_registry_variables variables += project.deployment_variables if has_environment? + variables += project.auto_devops_variables variables += yaml_variables variables += user_variables variables += project.group.secret_variables_for(ref, project).map(&:to_runner_variable) if project.group @@ -397,7 +400,9 @@ module Ci [ { key: 'GITLAB_USER_ID', value: user.id.to_s, public: true }, - { key: 'GITLAB_USER_EMAIL', value: user.email, public: true } + { key: 'GITLAB_USER_EMAIL', value: user.email, public: true }, + { key: 'GITLAB_USER_LOGIN', value: user.username, public: true }, + { key: 'GITLAB_USER_NAME', value: user.name, public: true } ] end @@ -456,6 +461,10 @@ module Ci trace end + def serializable_hash(options = {}) + super(options).merge(when: read_attribute(:when)) + end + private def update_artifacts_size diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ea7331cb27f..871c76fbad3 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -38,6 +38,7 @@ module Ci validates :status, presence: { unless: :importing? } validate :valid_commit_sha, unless: :importing? + after_initialize :set_config_source, if: :new_record? after_create :keep_around_commits, unless: :importing? enum source: { @@ -50,6 +51,12 @@ module Ci external: 6 } + enum config_source: { + unknown_source: nil, + repository_source: 1, + auto_devops_source: 2 + } + state_machine :status, initial: :created do event :enqueue do transition created: :pending @@ -304,6 +311,10 @@ module Ci @stage_seeds ||= config_processor.stage_seeds(self) end + def has_kubernetes_active? + project.kubernetes_service&.active? + end + def has_stage_seeds? stage_seeds.any? end @@ -312,6 +323,14 @@ module Ci builds.latest.failed_but_allowed.any? end + def set_config_source + if ci_yaml_from_repo + self.config_source = :repository_source + elsif implied_ci_yaml_file + self.config_source = :auto_devops_source + end + end + def config_processor return unless ci_yaml_file return @config_processor if defined?(@config_processor) @@ -338,11 +357,17 @@ module Ci def ci_yaml_file return @ci_yaml_file if defined?(@ci_yaml_file) - @ci_yaml_file = begin - project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path) - rescue Rugged::ReferenceError, GRPC::NotFound, GRPC::Internal - self.yaml_errors = - "Failed to load CI/CD config file at #{ci_yaml_file_path}" + @ci_yaml_file = + if auto_devops_source? + implied_ci_yaml_file + else + ci_yaml_from_repo + end + + if @ci_yaml_file + @ci_yaml_file + else + self.yaml_errors = "Failed to load CI/CD config file for #{sha}" nil end end @@ -393,7 +418,8 @@ module Ci def predefined_variables [ { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }, - { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true } + { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true }, + { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true } ] end @@ -429,6 +455,23 @@ module Ci private + def ci_yaml_from_repo + return unless project + return unless sha + + project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path) + rescue GRPC::NotFound, Rugged::ReferenceError, GRPC::Internal + nil + end + + def implied_ci_yaml_file + return unless project + + if project.auto_devops_enabled? + Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content + end + end + def pipeline_data Gitlab::DataBuilder::Pipeline.build(self) end diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 085eeeae157..e7e02587759 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -9,7 +9,7 @@ module Ci belongs_to :owner, class_name: 'User' has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline' has_many :pipelines - has_many :variables, class_name: 'Ci::PipelineScheduleVariable' + has_many :variables, class_name: 'Ci::PipelineScheduleVariable', validate: false validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? } validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? } diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb index 1ff177616e8..ee5b8733fac 100644 --- a/app/models/ci/pipeline_schedule_variable.rb +++ b/app/models/ci/pipeline_schedule_variable.rb @@ -4,5 +4,7 @@ module Ci include HasVariable belongs_to :pipeline_schedule + + validates :key, uniqueness: { scope: :pipeline_schedule_id } end end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index c6d23898560..b1798084787 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -5,7 +5,7 @@ module Ci RUNNER_QUEUE_EXPIRY_TIME = 60.minutes ONLINE_CONTACT_TIMEOUT = 1.hour AVAILABLE_SCOPES = %w[specific shared active paused online].freeze - FORM_EDITABLE = %i[description tag_list active run_untagged locked].freeze + FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level].freeze has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -35,11 +35,17 @@ module Ci end validate :tag_constraints + validates :access_level, presence: true acts_as_taggable after_destroy :cleanup_runner_queue + enum access_level: { + not_protected: 0, + ref_protected: 1 + } + # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL and LIKE on MySQL. @@ -106,6 +112,8 @@ module Ci end def can_pick?(build) + return false if self.ref_protected? && !build.protected? + assignable_for?(build.project) && accepting_tags?(build) end @@ -142,7 +150,7 @@ module Ci expire: RUNNER_QUEUE_EXPIRY_TIME, overwrite: false) end - def is_runner_queue_value_latest?(value) + def runner_queue_value_latest?(value) ensure_runner_queue_value == value if value.present? end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 59570924c8d..754c37518b3 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -1,11 +1,70 @@ module Ci class Stage < ActiveRecord::Base extend Ci::Model + include Importable + include HasStatus + include Gitlab::OptimisticLocking + + enum status: HasStatus::STATUSES_ENUM belongs_to :project belongs_to :pipeline - has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id - has_many :builds, foreign_key: :commit_id + has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id + has_many :builds, foreign_key: :stage_id + + validates :project, presence: true, unless: :importing? + validates :pipeline, presence: true, unless: :importing? + validates :name, presence: true, unless: :importing? + + after_initialize do |stage| + self.status = DEFAULT_STATUS if self.status.nil? + end + + state_machine :status, initial: :created do + event :enqueue do + transition created: :pending + transition [:success, :failed, :canceled, :skipped] => :running + end + + event :run do + transition any - [:running] => :running + end + + event :skip do + transition any - [:skipped] => :skipped + end + + event :drop do + transition any - [:failed] => :failed + end + + event :succeed do + transition any - [:success] => :success + end + + event :cancel do + transition any - [:canceled] => :canceled + end + + event :block do + transition any - [:manual] => :manual + end + end + + def update_status + retry_optimistic_lock(self) do + case statuses.latest.status + when 'pending' then enqueue + when 'running' then run + when 'success' then succeed + when 'failed' then drop + when 'canceled' then cancel + when 'manual' then block + when 'skipped' then skip + else skip + end + end + end end end diff --git a/app/models/ci/trigger_request.rb b/app/models/ci/trigger_request.rb index c58ce5c3717..2c860598281 100644 --- a/app/models/ci/trigger_request.rb +++ b/app/models/ci/trigger_request.rb @@ -6,6 +6,10 @@ module Ci belongs_to :pipeline, foreign_key: :commit_id has_many :builds + # We switched to Ci::PipelineVariable from Ci::TriggerRequest.variables. + # Ci::TriggerRequest doesn't save variables anymore. + validates :variables, absence: true + serialize :variables # rubocop:disable Cop/ActiveRecordSerialize def user_variables diff --git a/app/models/commit.rb b/app/models/commit.rb index 7940733f557..2ae8890c1b3 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -16,6 +16,8 @@ class Commit participant :notes_with_associations attr_accessor :project, :author + attr_accessor :redacted_description_html + attr_accessor :redacted_title_html DIFF_SAFE_LINES = Gitlab::Git::DiffCollection::DEFAULT_LIMITS[:max_lines] @@ -26,6 +28,13 @@ class Commit # The SHA can be between 7 and 40 hex characters. COMMIT_SHA_PATTERN = '\h{7,40}'.freeze + def banzai_render_context(field) + context = { pipeline: :single_line, project: self.project } + context[:author] = self.author if self.author + + context + end + class << self def decorate(commits, project) commits.map do |commit| @@ -55,7 +64,8 @@ class Commit end def from_hash(hash, project) - new(Gitlab::Git::Commit.new(hash), project) + raw_commit = Gitlab::Git::Commit.new(project.repository.raw, hash) + new(raw_commit, project) end def valid_hash?(key) @@ -199,7 +209,7 @@ class Commit end def method_missing(m, *args, &block) - @raw.send(m, *args, &block) + @raw.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(method, include_private = false) @@ -250,6 +260,28 @@ class Commit project.repository.next_branch("cherry-pick-#{short_id}", mild: true) end + def cherry_pick_description(user) + message_body = "(cherry picked from commit #{sha})" + + if merged_merge_request?(user) + commits_in_merge_request = merged_merge_request(user).commits + + if commits_in_merge_request.present? + message_body << "\n" + + commits_in_merge_request.reverse.each do |commit_in_merge| + message_body << "\n#{commit_in_merge.short_id} #{commit_in_merge.title}" + end + end + end + + message_body + end + + def cherry_pick_message(user) + %Q{#{message}\n\n#{cherry_pick_description(user)}} + end + def revert_description(user) if merged_merge_request?(user) "This reverts merge request #{merged_merge_request(user).to_reference}" @@ -320,21 +352,11 @@ class Commit end def raw_diffs(*args) - if Gitlab::GitalyClient.feature_enabled?(:commit_raw_diffs) - Gitlab::GitalyClient::CommitService.new(project.repository).diff_from_parent(self, *args) - else - raw.diffs(*args) - end + raw.diffs(*args) end def raw_deltas - @deltas ||= Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::CommitService.new(project.repository).commit_deltas(self) - else - raw.deltas - end - end + @deltas ||= raw.deltas end def diffs(diff_options = nil) diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 07cec63b939..f3888528940 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -38,15 +38,23 @@ class CommitStatus < ActiveRecord::Base scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } - state_machine :status do - event :enqueue do - transition [:created, :skipped, :manual] => :pending - end + enum failure_reason: { + unknown_failure: nil, + script_failure: 1, + api_failure: 2, + stuck_or_timeout_failure: 3, + runner_system_failure: 4 + } + state_machine :status do event :process do transition [:skipped, :manual] => :created end + event :enqueue do + transition [:created, :skipped, :manual] => :pending + end + event :run do transition pending: :running end @@ -79,6 +87,11 @@ class CommitStatus < ActiveRecord::Base commit_status.finished_at = Time.now end + before_transition any => :failed do |commit_status, transition| + failure_reason = transition.args.first + commit_status.failure_reason = failure_reason + end + after_transition do |commit_status, transition| next if transition.loopback? @@ -91,6 +104,7 @@ class CommitStatus < ActiveRecord::Base end end + StageUpdateWorker.perform_async(commit_status.stage_id) ExpireJobCacheWorker.perform_async(commit_status.id) end end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index f4f9b037957..9adc309a22b 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -11,6 +11,21 @@ module Awardable end module ClassMethods + def awarded(user, name) + sql = <<~EOL + EXISTS ( + SELECT TRUE + FROM award_emoji + WHERE user_id = :user_id AND + name = :name AND + awardable_type = :awardable_type AND + awardable_id = #{self.arel_table.name}.id + ) + EOL + + where(sql, user_id: user.id, name: name, awardable_type: self.name) + end + def order_upvotes_desc order_votes_desc(AwardEmoji::UPVOTE_NAME) end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 48547a938fc..193e459977a 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -78,7 +78,7 @@ module CacheMarkdownField def cached_html_up_to_date?(markdown_field) html_field = cached_markdown_fields.html_field(markdown_field) - cached = cached_html_for(markdown_field).present? && __send__(markdown_field).present? + cached = cached_html_for(markdown_field).present? && __send__(markdown_field).present? # rubocop:disable GitlabSecurity/PublicSend return false unless cached markdown_changed = attribute_changed?(markdown_field) || false @@ -93,14 +93,14 @@ module CacheMarkdownField end def attribute_invalidated?(attr) - __send__("#{attr}_invalidated?") + __send__("#{attr}_invalidated?") # rubocop:disable GitlabSecurity/PublicSend end def cached_html_for(markdown_field) raise ArgumentError.new("Unknown field: #{field}") unless cached_markdown_fields.markdown_fields.include?(markdown_field) - __send__(cached_markdown_fields.html_field(markdown_field)) + __send__(cached_markdown_fields.html_field(markdown_field)) # rubocop:disable GitlabSecurity/PublicSend end included do diff --git a/app/models/concerns/editable.rb b/app/models/concerns/editable.rb index 28623d257a6..c0a3099f676 100644 --- a/app/models/concerns/editable.rb +++ b/app/models/concerns/editable.rb @@ -1,7 +1,7 @@ module Editable extend ActiveSupport::Concern - def is_edited? + def edited? last_edited_at.present? && last_edited_at != created_at end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 32af5566135..3803e18a96e 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -8,6 +8,8 @@ module HasStatus ACTIVE_STATUSES = %w[pending running].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze + STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze class_methods do def status_sql diff --git a/app/models/concerns/internal_id.rb b/app/models/concerns/internal_id.rb index 67a0adfcd56..a3d0ac8d862 100644 --- a/app/models/concerns/internal_id.rb +++ b/app/models/concerns/internal_id.rb @@ -9,7 +9,7 @@ module InternalId def set_iid if iid.blank? parent = project || group - records = parent.send(self.class.name.tableize) + records = parent.public_send(self.class.name.tableize) # rubocop:disable GitlabSecurity/PublicSend records = records.with_deleted if self.paranoid? max_iid = records.maximum(:iid) diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 3731b7c8577..265f6e48540 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -6,6 +6,7 @@ # module Issuable extend ActiveSupport::Concern + include Gitlab::SQL::Pattern include CacheMarkdownField include Participable include Mentionable @@ -122,7 +123,9 @@ module Issuable # # Returns an ActiveRecord::Relation. def search(query) - where(arel_table[:title].matches("%#{query}%")) + title = to_fuzzy_arel(:title, query) + + where(title) end # Searches for records with a matching title or description. @@ -133,10 +136,10 @@ module Issuable # # Returns an ActiveRecord::Relation. def full_search(query) - t = arel_table - pattern = "%#{query}%" + title = to_fuzzy_arel(:title, query) + description = to_fuzzy_arel(:description, query) - where(t[:title].matches(pattern).or(t[:description].matches(pattern))) + where(title&.or(description)) end def sort(method, excluded_labels: []) @@ -331,4 +334,11 @@ module Issuable metrics = self.metrics || create_metrics metrics.record! end + + ## + # Override in issuable specialization + # + def first_contribution? + false + end end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index c034bf9cbc0..1db6b2d2fa2 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -56,7 +56,7 @@ module Mentionable end self.class.mentionable_attrs.each do |attr, options| - text = __send__(attr) + text = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend options = options.merge( cache_key: [self, attr], author: author, @@ -100,7 +100,7 @@ module Mentionable end self.class.mentionable_attrs.any? do |attr, _| - __send__(attr) =~ reference_pattern + __send__(attr) =~ reference_pattern # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index f0998465822..710fc1ed647 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -70,19 +70,19 @@ module Milestoneish due_date && due_date.past? end - def is_group_milestone? + def group_milestone? false end - def is_project_milestone? + def project_milestone? false end - def is_legacy_group_milestone? + def legacy_group_milestone? false end - def is_dashboard_milestone? + def dashboard_milestone? false end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index c7bdc997eca..1c4ddabcad5 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -24,6 +24,10 @@ module Noteable DiscussionNote::NOTEABLE_TYPES.include?(base_class_name) end + def discussions_rendered_on_frontend? + false + end + def discussion_notes notes end @@ -38,7 +42,7 @@ module Noteable def grouped_diff_discussions(*args) # Doesn't use `discussion_notes`, because this may include commit diff notes - # besides MR diff notes, that we do no want to display on the MR Changes tab. + # besides MR diff notes, that we do not want to display on the MR Changes tab. notes.inc_relations_for_view.grouped_diff_discussions(*args) end diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index 4865c0a14b1..ce69fd34ac5 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -82,7 +82,7 @@ module Participable if attr.respond_to?(:call) source.instance_exec(current_user, ext, &attr) else - process << source.__send__(attr) + process << source.__send__(attr) # rubocop:disable GitlabSecurity/PublicSend end end when Enumerable, ActiveRecord::Relation diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index 60734bc6660..cb59b4da3d7 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -32,6 +32,6 @@ module ProjectFeaturesCompatibility build_project_feature unless project_feature access_level = Gitlab::Utils.to_boolean(value) ? ProjectFeature::ENABLED : ProjectFeature::DISABLED - project_feature.send(:write_attribute, field, access_level) + project_feature.__send__(:write_attribute, field, access_level) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index ef95d6b0f98..454374121f3 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -23,7 +23,7 @@ module ProtectedRef # If we don't `protected_branch` or `protected_tag` would be empty and # `project` cannot be delegated to it, which in turn would cause validations # to fail. - has_many :"#{type}_access_levels", dependent: :destroy, inverse_of: self.model_name.singular # rubocop:disable Cop/ActiveRecordDependent + has_many :"#{type}_access_levels", inverse_of: self.model_name.singular # rubocop:disable Cop/ActiveRecordDependent validates :"#{type}_access_levels", length: { is: 1, message: "are restricted to a single instance per #{self.model_name.human}." } diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb index 10f4be72016..78ac4f324e7 100644 --- a/app/models/concerns/referable.rb +++ b/app/models/concerns/referable.rb @@ -25,6 +25,11 @@ module Referable to_reference(from_project) end + included do + alias_method :non_referable_inspect, :inspect + alias_method :inspect, :referable_inspect + end + def referable_inspect if respond_to?(:id) "#<#{self.class.name} id:#{id} #{to_reference(full: true)}>" @@ -33,10 +38,6 @@ module Referable end end - def inspect - referable_inspect - end - module ClassMethods # The character that prefixes the actual reference identifier # diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index 7cb9a28a284..e961c97e337 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -10,8 +10,12 @@ module RelativePositioning after_save :save_positionable_neighbours end + def project_ids + [project.id] + end + def max_relative_position - self.class.in_projects(project.id).maximum(:relative_position) + self.class.in_projects(project_ids).maximum(:relative_position) end def prev_relative_position @@ -19,7 +23,7 @@ module RelativePositioning if self.relative_position prev_pos = self.class - .in_projects(project.id) + .in_projects(project_ids) .where('relative_position < ?', self.relative_position) .maximum(:relative_position) end @@ -32,7 +36,7 @@ module RelativePositioning if self.relative_position next_pos = self.class - .in_projects(project.id) + .in_projects(project_ids) .where('relative_position > ?', self.relative_position) .minimum(:relative_position) end @@ -59,7 +63,7 @@ module RelativePositioning pos_after = before.next_relative_position if before.shift_after? - issue_to_move = self.class.in_projects(project.id).find_by!(relative_position: pos_after) + issue_to_move = self.class.in_projects(project_ids).find_by!(relative_position: pos_after) issue_to_move.move_after @positionable_neighbours = [issue_to_move] @@ -74,7 +78,7 @@ module RelativePositioning pos_before = after.prev_relative_position if after.shift_before? - issue_to_move = self.class.in_projects(project.id).find_by!(relative_position: pos_before) + issue_to_move = self.class.in_projects(project_ids).find_by!(relative_position: pos_before) issue_to_move.move_before @positionable_neighbours = [issue_to_move] diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index dd979e7bb17..f006a271327 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -24,6 +24,7 @@ module ResolvableDiscussion delegate :resolved_at, :resolved_by, + :resolved_by_push?, to: :last_resolved_note, allow_nil: true diff --git a/app/models/concerns/resolvable_note.rb b/app/models/concerns/resolvable_note.rb index 05eb6f86704..668c5a079e3 100644 --- a/app/models/concerns/resolvable_note.rb +++ b/app/models/concerns/resolvable_note.rb @@ -51,22 +51,34 @@ module ResolvableNote end # If you update this method remember to also update `.resolve!` - def resolve!(current_user) - return unless resolvable? - return if resolved? + def resolve_without_save(current_user, resolved_by_push: false) + return false unless resolvable? + return false if resolved? self.resolved_at = Time.now self.resolved_by = current_user - save! + self.resolved_by_push = resolved_by_push + + true end # If you update this method remember to also update `.unresolve!` - def unresolve! - return unless resolvable? - return unless resolved? + def unresolve_without_save + return false unless resolvable? + return false unless resolved? self.resolved_at = nil self.resolved_by = nil - save! + + true + end + + def resolve!(current_user, resolved_by_push: false) + resolve_without_save(current_user, resolved_by_push: resolved_by_push) && + save! + end + + def unresolve! + unresolve_without_save && save! end end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index a155a064032..db3cd257584 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -6,10 +6,6 @@ module Sortable extend ActiveSupport::Concern included do - # By default all models should be ordered - # by created_at field starting from newest - default_scope { order_id_desc } - scope :order_id_desc, -> { reorder(id: :desc) } scope :order_id_asc, -> { reorder(id: :asc) } scope :order_created_desc, -> { reorder(created_at: :desc) } diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index bd75f25a210..731d9b9a745 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -28,7 +28,7 @@ module Spammable def submittable_as_spam? if user_agent_detail - user_agent_detail.submittable? && current_application_settings.akismet_enabled + user_agent_detail.submittable? && Gitlab::CurrentSettings.current_application_settings.akismet_enabled else false end @@ -58,7 +58,7 @@ module Spammable options.fetch(:spam_title, false) end - public_send(attr.first) if attr && respond_to?(attr.first.to_sym) + public_send(attr.first) if attr && respond_to?(attr.first.to_sym) # rubocop:disable GitlabSecurity/PublicSend end def spam_description @@ -66,12 +66,12 @@ module Spammable options.fetch(:spam_description, false) end - public_send(attr.first) if attr && respond_to?(attr.first.to_sym) + public_send(attr.first) if attr && respond_to?(attr.first.to_sym) # rubocop:disable GitlabSecurity/PublicSend end def spammable_text result = self.class.spammable_attrs.map do |attr| - public_send(attr.first) + public_send(attr.first) # rubocop:disable GitlabSecurity/PublicSend end result.reject(&:blank?).join("\n") diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb deleted file mode 100644 index 815db712285..00000000000 --- a/app/models/concerns/storage/legacy_project.rb +++ /dev/null @@ -1,76 +0,0 @@ -module Storage - module LegacyProject - extend ActiveSupport::Concern - - def disk_path - full_path - end - - def ensure_storage_path_exist - gitlab_shell.add_namespace(repository_storage_path, namespace.full_path) - end - - def rename_repo - path_was = previous_changes['path'].first - old_path_with_namespace = File.join(namespace.full_path, path_was) - new_path_with_namespace = File.join(namespace.full_path, path) - - Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - - if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - - # we currently doesn't support renaming repository if it contains images in container registry - raise StandardError.new('Project cannot be renamed, because images are present in its container registry') - end - - expire_caches_before_rename(old_path_with_namespace) - - if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) - # If repository moved successfully we need to send update instructions to users. - # However we cannot allow rollback since we moved repository - # So we basically we mute exceptions in next actions - begin - gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") - send_move_instructions(old_path_with_namespace) - expires_full_path_cache - - @old_path_with_namespace = old_path_with_namespace - - SystemHooksService.new.execute_hooks_for(self, :rename) - - @repository = nil - rescue => e - Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false - end - else - Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise StandardError.new('repository cannot be renamed') - end - - Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) - Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) - end - - def create_repository(force: false) - # Forked import is handled asynchronously - return if forked? && !force - - if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) - repository.after_create - true - else - errors.add(:base, 'Failed to create repository via gitlab-shell') - false - end - end - end -end diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 1ca7f91dc03..a7d5de48c66 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -44,7 +44,8 @@ module TokenAuthenticatable end define_method("ensure_#{token_field}!") do - send("reset_#{token_field}!") if read_attribute(token_field).blank? + send("reset_#{token_field}!") if read_attribute(token_field).blank? # rubocop:disable GitlabSecurity/PublicSend + read_attribute(token_field) end diff --git a/app/models/dashboard_milestone.rb b/app/models/dashboard_milestone.rb index fac7c5e5c85..86eb4ec76fc 100644 --- a/app/models/dashboard_milestone.rb +++ b/app/models/dashboard_milestone.rb @@ -3,7 +3,7 @@ class DashboardMilestone < GlobalMilestone { authorized_only: true } end - def is_dashboard_milestone? + def dashboard_milestone? true end end diff --git a/app/models/deploy_keys_project.rb b/app/models/deploy_keys_project.rb index ae8486bd9ac..b37b9bfbdac 100644 --- a/app/models/deploy_keys_project.rb +++ b/app/models/deploy_keys_project.rb @@ -12,7 +12,7 @@ class DeployKeysProject < ActiveRecord::Base def destroy_orphaned_deploy_key return unless self.deploy_key.destroyed_when_orphaned? && self.deploy_key.orphaned? - + self.deploy_key.destroy end end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 056c49e7162..7bcded5b5e1 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -49,7 +49,7 @@ class Deployment < ActiveRecord::Base # created before then could have a `sha` referring to a commit that no # longer exists in the repository, so just ignore those. begin - project.repository.is_ancestor?(commit.id, sha) + project.repository.ancestor?(commit.id, sha) rescue Rugged::OdbError false end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index d1cec7613af..b80da7b246a 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -81,6 +81,10 @@ class Discussion last_note.author end + def updated? + last_updated_at != created_at + end + def id first_note.discussion_id(context_noteable) end diff --git a/app/models/environment.rb b/app/models/environment.rb index e9ebf0637f3..435eeaf0e2e 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -114,7 +114,7 @@ class Environment < ActiveRecord::Base end def ref_path - "refs/environments/#{Shellwords.shellescape(name)}" + "refs/#{Repository::REF_ENVIRONMENTS}/#{Shellwords.shellescape(name)}" end def formatted_external_url diff --git a/app/models/event.rb b/app/models/event.rb index 8d93a228494..c313bbb66f8 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,5 +1,6 @@ class Event < ActiveRecord::Base include Sortable + include IgnorableColumn default_scope { reorder(nil).where.not(author_id: nil) } CREATED = 1 @@ -48,9 +49,7 @@ class Event < ActiveRecord::Base belongs_to :author, class_name: "User" belongs_to :project belongs_to :target, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations - - # For Hash only - serialize :data # rubocop:disable Cop/ActiveRecordSerialize + has_one :push_event_payload, foreign_key: :event_id # Callbacks after_create :reset_project_activity @@ -60,14 +59,53 @@ class Event < ActiveRecord::Base scope :recent, -> { reorder(id: :desc) } scope :code_push, -> { where(action: PUSHED) } - scope :in_projects, ->(projects) do - where(project_id: projects.pluck(:id)).recent + scope :in_projects, -> (projects) do + sub_query = projects + .except(:order) + .select(1) + .where('projects.id = events.project_id') + + where('EXISTS (?)', sub_query).recent + end + + scope :with_associations, -> do + # We're using preload for "push_event_payload" as otherwise the association + # is not always available (depending on the query being built). + includes(:author, :project, project: :namespace) + .preload(:target, :push_event_payload) end - scope :with_associations, -> { includes(:author, :project, project: :namespace).preload(:target) } scope :for_milestone_id, ->(milestone_id) { where(target_type: "Milestone", target_id: milestone_id) } + self.inheritance_column = 'action' + + # "data" will be removed in 10.0 but it may be possible that JOINs happen that + # include this column, hence we're ignoring it as well. + ignore_column :data + class << self + def model_name + ActiveModel::Name.new(self, nil, 'event') + end + + def find_sti_class(action) + if action.to_i == PUSHED + PushEvent + else + Event + end + end + + def subclass_from_attributes(attrs) + # Without this Rails will keep calling this method on the returned class, + # resulting in an infinite loop. + return unless self == Event + + action = attrs.with_indifferent_access[inheritance_column].to_i + + PushEvent if action == PUSHED + end + # Update Gitlab::ContributionsCalendar#activity_dates if this changes def contributions where("action = ? OR (target_type IN (?) AND action IN (?)) OR (target_type = ? AND action = ?)", @@ -122,7 +160,7 @@ class Event < ActiveRecord::Base end def push? - action == PUSHED && valid_push? + false end def merged? @@ -235,77 +273,6 @@ class Event < ActiveRecord::Base end end - def valid_push? - data[:ref] && ref_name.present? - rescue - false - end - - def tag? - Gitlab::Git.tag_ref?(data[:ref]) - end - - def branch? - Gitlab::Git.branch_ref?(data[:ref]) - end - - def new_ref? - Gitlab::Git.blank_ref?(commit_from) - end - - def rm_ref? - Gitlab::Git.blank_ref?(commit_to) - end - - def md_ref? - !(rm_ref? || new_ref?) - end - - def commit_from - data[:before] - end - - def commit_to - data[:after] - end - - def ref_name - if tag? - tag_name - else - branch_name - end - end - - def branch_name - @branch_name ||= Gitlab::Git.ref_name(data[:ref]) - end - - def tag_name - @tag_name ||= Gitlab::Git.ref_name(data[:ref]) - end - - # Max 20 commits from push DESC - def commits - @commits ||= (data[:commits] || []).reverse - end - - def commits_count - data[:total_commits_count] || commits.count || 0 - end - - def ref_type - tag? ? "tag" : "branch" - end - - def push_with_commits? - !commits.empty? && commit_from && commit_to - end - - def last_push_to_non_root? - branch? && project.default_branch != branch_name - end - def target_iid target.respond_to?(:iid) ? target.iid : target_id end @@ -359,7 +326,7 @@ class Event < ActiveRecord::Base def body? if push? - push_with_commits? || rm_ref? + push_with_commits? elsif note? true else @@ -385,6 +352,12 @@ class Event < ActiveRecord::Base user ? author_id == user.id : false end + def to_partial_path + # We are intentionally using `Event` rather than `self.class` so that + # subclasses also use the `Event` implementation. + Event._to_partial_path + end + private def recent_update? diff --git a/app/models/event_collection.rb b/app/models/event_collection.rb new file mode 100644 index 00000000000..8b8244314af --- /dev/null +++ b/app/models/event_collection.rb @@ -0,0 +1,98 @@ +# A collection of events to display in an event list. +# +# An EventCollection is meant to be used for displaying events to a user (e.g. +# in a controller), it's not suitable for building queries that are used for +# building other queries. +class EventCollection + # To prevent users from putting too much pressure on the database by cycling + # through thousands of events we put a limit on the number of pages. + MAX_PAGE = 10 + + # projects - An ActiveRecord::Relation object that returns the projects for + # which to retrieve events. + # filter - An EventFilter instance to use for filtering events. + def initialize(projects, limit: 20, offset: 0, filter: nil) + @projects = projects + @limit = limit + @offset = offset + @filter = filter + end + + # Returns an Array containing the events. + def to_a + return [] if current_page > MAX_PAGE + + relation = if Gitlab::Database.join_lateral_supported? + relation_with_join_lateral + else + relation_without_join_lateral + end + + relation.with_associations.to_a + end + + private + + # Returns the events relation to use when JOIN LATERAL is not supported. + # + # This relation simply gets all the events for all authorized projects, then + # limits that set. + def relation_without_join_lateral + events = filtered_events.in_projects(projects) + + paginate_events(events) + end + + # Returns the events relation to use when JOIN LATERAL is supported. + # + # This relation is built using JOIN LATERAL, producing faster queries than a + # regular LIMIT + OFFSET approach. + def relation_with_join_lateral + projects_for_lateral = projects.select(:id).to_sql + + lateral = filtered_events + .limit(limit_for_join_lateral) + .where('events.project_id = projects_for_lateral.id') + .to_sql + + # The outer query does not need to re-apply the filters since the JOIN + # LATERAL body already takes care of this. + outer = base_relation + .from("(#{projects_for_lateral}) projects_for_lateral") + .joins("JOIN LATERAL (#{lateral}) AS #{Event.table_name} ON true") + + paginate_events(outer) + end + + def filtered_events + @filter ? @filter.apply_filter(base_relation) : base_relation + end + + def paginate_events(events) + events.limit(@limit).offset(@offset) + end + + def base_relation + # We want to have absolute control over the event queries being built, thus + # we're explicitly opting out of any default scopes that may be set. + Event.unscoped.recent + end + + def limit_for_join_lateral + # Applying the OFFSET on the inside of a JOIN LATERAL leads to incorrect + # results. To work around this we need to increase the inner limit for every + # page. + # + # This means that on page 1 we use LIMIT 20, and an outer OFFSET of 0. On + # page 2 we use LIMIT 40 and an outer OFFSET of 20. + @limit + @offset + end + + def current_page + (@offset / @limit) + 1 + end + + def projects + @projects.except(:order) + end +end diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb index 3df60ddc950..1633acd4fa9 100644 --- a/app/models/gpg_key.rb +++ b/app/models/gpg_key.rb @@ -56,7 +56,7 @@ class GpgKey < ActiveRecord::Base def verified_user_infos user_infos.select do |user_info| - user_info[:email] == user.email + user.verified_email?(user_info[:email]) end end @@ -64,13 +64,17 @@ class GpgKey < ActiveRecord::Base user_infos.map do |user_info| [ user_info[:email], - user_info[:email] == user.email + user.verified_email?(user_info[:email]) ] end.to_h end def verified? - emails_with_verified_status.any? { |_email, verified| verified } + emails_with_verified_status.values.any? + end + + def verified_and_belongs_to_email?(email) + emails_with_verified_status.fetch(email, false) end def update_invalid_gpg_signatures @@ -78,11 +82,14 @@ class GpgKey < ActiveRecord::Base end def revoke - GpgSignature.where(gpg_key: self, valid_signature: true).update_all( - gpg_key_id: nil, - valid_signature: false, - updated_at: Time.zone.now - ) + GpgSignature + .where(gpg_key: self) + .where.not(verification_status: GpgSignature.verification_statuses[:unknown_key]) + .update_all( + gpg_key_id: nil, + verification_status: GpgSignature.verification_statuses[:unknown_key], + updated_at: Time.zone.now + ) destroy end diff --git a/app/models/gpg_signature.rb b/app/models/gpg_signature.rb index 1ac0e123ff1..454c90d5fc4 100644 --- a/app/models/gpg_signature.rb +++ b/app/models/gpg_signature.rb @@ -1,9 +1,21 @@ class GpgSignature < ActiveRecord::Base include ShaAttribute + include IgnorableColumn + + ignore_column :valid_signature sha_attribute :commit_sha sha_attribute :gpg_key_primary_keyid + enum verification_status: { + unverified: 0, + verified: 1, + same_user_different_email: 2, + other_user: 3, + unverified_key: 4, + unknown_key: 5 + } + belongs_to :project belongs_to :gpg_key @@ -18,4 +30,8 @@ class GpgSignature < ActiveRecord::Base def commit project.commit(commit_sha) end + + def gpg_commit + Gitlab::Gpg::Commit.new(commit) + end end diff --git a/app/models/group.rb b/app/models/group.rb index bd5735ed82e..e746e4a12c9 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -16,6 +16,7 @@ class Group < Namespace source: :user has_many :requesters, -> { where.not(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' # rubocop:disable Cop/ActiveRecordDependent + has_many :members_and_requesters, as: :source, class_name: 'GroupMember' has_many :milestones has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -26,6 +27,8 @@ class Group < Namespace validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :visibility_level_allowed_by_projects + validate :visibility_level_allowed_by_sub_groups + validate :visibility_level_allowed_by_parent validates :avatar, file_size: { maximum: 200.kilobytes.to_i } @@ -102,15 +105,24 @@ class Group < Namespace full_name end - def visibility_level_allowed_by_projects - allowed_by_projects = self.projects.where('visibility_level > ?', self.visibility_level).none? + def visibility_level_allowed_by_parent?(level = self.visibility_level) + return true unless parent_id && parent_id.nonzero? - unless allowed_by_projects - level_name = Gitlab::VisibilityLevel.level_name(visibility_level).downcase - self.errors.add(:visibility_level, "#{level_name} is not allowed since there are projects with higher visibility.") - end + level <= parent.visibility_level + end + + def visibility_level_allowed_by_projects?(level = self.visibility_level) + !projects.where('visibility_level > ?', level).exists? + end + + def visibility_level_allowed_by_sub_groups?(level = self.visibility_level) + !children.where('visibility_level > ?', level).exists? + end - allowed_by_projects + def visibility_level_allowed?(level = self.visibility_level) + visibility_level_allowed_by_parent?(level) && + visibility_level_allowed_by_projects?(level) && + visibility_level_allowed_by_sub_groups?(level) end def avatar_url(**args) @@ -206,27 +218,45 @@ class Group < Namespace SystemHooksService.new end - def refresh_members_authorized_projects + def refresh_members_authorized_projects(blocking: true) UserProjectAccessChangedService.new(user_ids_for_project_authorizations) - .execute + .execute(blocking: blocking) end def user_ids_for_project_authorizations - users_with_parents.pluck(:id) + members_with_parents.pluck(:user_id) end def members_with_parents - GroupMember.active.where(source_id: ancestors.pluck(:id).push(id)).where.not(user_id: nil) + # Avoids an unnecessary SELECT when the group has no parents + source_ids = + if parent_id + self_and_ancestors.reorder(nil).select(:id) + else + id + end + + GroupMember + .active_without_invites + .where(source_id: source_ids) + end + + def members_with_descendants + GroupMember + .active_without_invites + .where(source_id: self_and_descendants.reorder(nil).select(:id)) end def users_with_parents - User.where(id: members_with_parents.select(:user_id)) + User + .where(id: members_with_parents.select(:user_id)) + .reorder(nil) end def users_with_descendants - members_with_descendants = GroupMember.non_request.where(source_id: descendants.pluck(:id).push(id)) - - User.where(id: members_with_descendants.select(:user_id)) + User + .where(id: members_with_descendants.select(:user_id)) + .reorder(nil) end def max_member_access_for_user(user) @@ -257,11 +287,29 @@ class Group < Namespace list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten end - protected + private def update_two_factor_requirement return unless require_two_factor_authentication_changed? || two_factor_grace_period_changed? users.find_each(&:update_two_factor_requirement) end + + def visibility_level_allowed_by_parent + return if visibility_level_allowed_by_parent? + + errors.add(:visibility_level, "#{visibility} is not allowed since the parent group has a #{parent.visibility} visibility.") + end + + def visibility_level_allowed_by_projects + return if visibility_level_allowed_by_projects? + + errors.add(:visibility_level, "#{visibility} is not allowed since this group contains projects with higher visibility.") + end + + def visibility_level_allowed_by_sub_groups + return if visibility_level_allowed_by_sub_groups? + + errors.add(:visibility_level, "#{visibility} is not allowed since there are sub-groups with higher visibility.") + end end diff --git a/app/models/group_milestone.rb b/app/models/group_milestone.rb index 65249bd7bfc..98135ee3c8b 100644 --- a/app/models/group_milestone.rb +++ b/app/models/group_milestone.rb @@ -17,7 +17,7 @@ class GroupMilestone < GlobalMilestone { group_id: group.id } end - def is_legacy_group_milestone? + def legacy_group_milestone? true end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 1c948c8957e..8c7d492e605 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -9,11 +9,8 @@ class Issue < ActiveRecord::Base include Spammable include FasterCacheKeys include RelativePositioning - include IgnorableColumn include CreatedAtFilterable - ignore_column :position - DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze @@ -53,7 +50,10 @@ class Issue < ActiveRecord::Base scope :preload_associations, -> { preload(:labels, project: :namespace) } + scope :public_only, -> { where(confidential: false) } + after_save :expire_etag_cache + after_commit :update_project_counter_caches, on: :destroy attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true @@ -269,6 +269,20 @@ class Issue < ActiveRecord::Base end end + def discussions_rendered_on_frontend? + true + end + + def update_project_counter_caches? + state_changed? || confidential_changed? + end + + def update_project_counter_caches + return unless update_project_counter_caches? + + Projects::OpenIssuesCountService.new(project).refresh_cache + end + private # Returns `true` if the given User can read the current Issue. diff --git a/app/models/key.rb b/app/models/key.rb index 49bc26122fa..a6b4dcfec0d 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -1,6 +1,7 @@ require 'digest/md5' class Key < ActiveRecord::Base + include Gitlab::CurrentSettings include Sortable LAST_USED_AT_REFRESH_TIME = 1.day.to_i @@ -12,14 +13,18 @@ class Key < ActiveRecord::Base validates :title, presence: true, length: { maximum: 255 } + validates :key, presence: true, length: { maximum: 5000 }, format: { with: /\A(ssh|ecdsa)-.*\Z/ } + validates :fingerprint, uniqueness: true, presence: { message: 'cannot be generated' } + validate :key_meets_restrictions + delegate :name, :email, to: :user, prefix: true after_commit :add_to_shell, on: :create @@ -80,6 +85,10 @@ class Key < ActiveRecord::Base SystemHooksService.new.execute_hooks_for(self, :destroy) end + def public_key + @public_key ||= Gitlab::SSHPublicKey.new(key) + end + private def generate_fingerprint @@ -87,7 +96,27 @@ class Key < ActiveRecord::Base return unless self.key.present? - self.fingerprint = Gitlab::KeyFingerprint.new(self.key).fingerprint + self.fingerprint = public_key.fingerprint + end + + def key_meets_restrictions + restriction = current_application_settings.key_restriction_for(public_key.type) + + if restriction == ApplicationSetting::FORBIDDEN_KEY_VALUE + errors.add(:key, forbidden_key_type_message) + elsif public_key.bits < restriction + errors.add(:key, "must be at least #{restriction} bits") + end + end + + def forbidden_key_type_message + allowed_types = + current_application_settings + .allowed_key_types + .map(&:upcase) + .to_sentence(last_word_connector: ', or ', two_words_connector: ' or ') + + "type is forbidden. Must be #{allowed_types}" end def notify_user diff --git a/app/models/label.rb b/app/models/label.rb index 674bb3f2720..958141a7358 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -34,7 +34,8 @@ class Label < ActiveRecord::Base scope :templates, -> { where(template: true) } scope :with_title, ->(title) { where(title: title) } - scope :on_project_boards, ->(project_id) { joins(lists: :board).merge(List.movable).where(boards: { project_id: project_id }) } + scope :with_lists_and_board, -> { joins(lists: :board).merge(List.movable) } + scope :on_project_boards, ->(project_id) { with_lists_and_board.where(boards: { project_id: project_id }) } def self.prioritized(project) joins(:priorities) @@ -172,6 +173,7 @@ class Label < ActiveRecord::Base def as_json(options = {}) super(options).tap do |json| + json[:type] = self.try(:type) json[:priority] = priority(options[:project]) if options.key?(:project) end end diff --git a/app/models/member.rb b/app/models/member.rb index dc9247bc9a0..cbbd58f2eaf 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -41,9 +41,20 @@ class Member < ActiveRecord::Base is_external_invite = arel_table[:user_id].eq(nil).and(arel_table[:invite_token].not_eq(nil)) user_is_active = User.arel_table[:state].eq(:active) - includes(:user).references(:users) - .where(is_external_invite.or(user_is_active)) + user_ok = Arel::Nodes::Grouping.new(is_external_invite).or(user_is_active) + + left_join_users + .where(user_ok) + .where(requested_at: nil) + .reorder(nil) + end + + # Like active, but without invites. For when a User is required. + scope :active_without_invites, -> do + left_join_users + .where(users: { state: 'active' }) .where(requested_at: nil) + .reorder(nil) end scope :invite, -> { where.not(invite_token: nil) } @@ -115,20 +126,11 @@ class Member < ActiveRecord::Base find_by(invite_token: invite_token) end - def add_user(source, user, access_level, current_user: nil, expires_at: nil) - user = retrieve_user(user) + def add_user(source, user, access_level, existing_members: nil, current_user: nil, expires_at: nil) + # `user` can be either a User object, User ID or an email to be invited + member = retrieve_member(source, user, existing_members) access_level = retrieve_access_level(access_level) - # `user` can be either a User object or an email to be invited - member = - if user.is_a?(User) - source.members.find_by(user_id: user.id) || - source.requesters.find_by(user_id: user.id) || - source.members.build(user_id: user.id) - else - source.members.build(invite_email: user) - end - return member unless can_update_member?(current_user, member) member.attributes = { @@ -154,17 +156,15 @@ class Member < ActiveRecord::Base def add_users(source, users, access_level, current_user: nil, expires_at: nil) return [] unless users.present? - # Collect all user ids into separate array - # so we can use single sql query to get user objects - user_ids = users.select { |user| user =~ /\A\d+\Z/ } - users = users - user_ids + User.where(id: user_ids) + emails, users, existing_members = parse_users_list(source, users) self.transaction do - users.map do |user| + (emails + users).map! do |user| add_user( source, user, access_level, + existing_members: existing_members, current_user: current_user, expires_at: expires_at ) @@ -178,6 +178,31 @@ class Member < ActiveRecord::Base private + def parse_users_list(source, list) + emails, user_ids, users = [], [], [] + existing_members = {} + + list.each do |item| + case item + when User + users << item + when Integer + user_ids << item + when /\A\d+\Z/ + user_ids << item.to_i + when Devise.email_regexp + emails << item + end + end + + if user_ids.present? + users.concat(User.where(id: user_ids)) + existing_members = source.members_and_requesters.where(user_id: user_ids).index_by(&:user_id) + end + + [emails, users, existing_members] + end + # This method is used to find users that have been entered into the "Add members" field. # These can be the User objects directly, their IDs, their emails, or new emails to be invited. def retrieve_user(user) @@ -186,6 +211,20 @@ class Member < ActiveRecord::Base User.find_by(id: user) || User.find_by(email: user) || user end + def retrieve_member(source, user, existing_members) + user = retrieve_user(user) + + if user.is_a?(User) + if existing_members + existing_members[user.id] || source.members.build(user_id: user.id) + else + source.members_and_requesters.find_or_initialize_by(user_id: user.id) + end + else + source.members.build(invite_email: user) + end + end + def retrieve_access_level(access_level) access_levels.fetch(access_level) { access_level.to_i } end @@ -276,6 +315,13 @@ class Member < ActiveRecord::Base @notification_setting ||= user.notification_settings_for(source) end + def notifiable?(type, opts = {}) + # always notify when there isn't a user yet + return true if user.blank? + + NotificationRecipientService.notifiable?(user, type, notifiable_options.merge(opts)) + end + private def send_invite @@ -332,4 +378,8 @@ class Member < ActiveRecord::Base def notification_service NotificationService.new end + + def notifiable_options + {} + end end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 47040f95533..661e668dbf9 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -30,6 +30,10 @@ class GroupMember < Member 'Group' end + def notifiable_options + { group: group } + end + private def send_invite diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index c0e17f4bfc8..b6f1dd272cd 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -87,6 +87,10 @@ class ProjectMember < Member project.owner == user end + def notifiable_options + { project: project } + end + private def delete_member_todos diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index e83b11f7668..2a56bab48a3 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -7,7 +7,6 @@ class MergeRequest < ActiveRecord::Base include IgnorableColumn include CreatedAtFilterable - ignore_column :position ignore_column :locked_at belongs_to :target_project, class_name: "Project" @@ -32,6 +31,7 @@ class MergeRequest < ActiveRecord::Base after_create :ensure_merge_request_diff, unless: :importing? after_update :reload_diff_if_branch_changed + after_commit :update_project_counter_caches, on: :destroy # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests @@ -162,7 +162,7 @@ class MergeRequest < ActiveRecord::Base target = unscoped.where(target_project_id: relation).select(:id) union = Gitlab::SQL::Union.new([source, target]) - where("merge_requests.id IN (#{union.to_sql})") + where("merge_requests.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze @@ -241,6 +241,14 @@ class MergeRequest < ActiveRecord::Base end end + # Calls `MergeWorker` to proceed with the merge process and + # updates `merge_jid` with the MergeWorker#jid. + # This helps tracking enqueued and ongoing merge jobs. + def merge_async(user_id, params) + jid = MergeWorker.perform_async(id, user_id, params) + update_column(:merge_jid, jid) + end + def first_commit merge_request_diff ? merge_request_diff.first_commit : compare_commits.first end @@ -384,9 +392,7 @@ class MergeRequest < ActiveRecord::Base end def merge_ongoing? - return false unless merge_jid - - Gitlab::SidekiqStatus.num_running([merge_jid]) > 0 + !!merge_jid && !merged? end def closed_without_fork? @@ -443,7 +449,8 @@ class MergeRequest < ActiveRecord::Base end def reload_diff_if_branch_changed - if source_branch_changed? || target_branch_changed? + if (source_branch_changed? || target_branch_changed?) && + (source_branch_head && target_branch_head) reload_diff end end @@ -598,6 +605,8 @@ class MergeRequest < ActiveRecord::Base self.merge_requests_closing_issues.delete_all closes_issues(current_user).each do |issue| + next if issue.is_a?(ExternalIssue) + self.merge_requests_closing_issues.create!(issue: issue) end end @@ -682,9 +691,8 @@ class MergeRequest < ActiveRecord::Base if !include_description && closes_issues_references.present? message << "Closes #{closes_issues_references.to_sentence}" end - message << "#{description}" if include_description && description.present? - message << "See merge request #{to_reference}" + message << "See merge request #{to_reference(full: true)}" message.join("\n\n") end @@ -792,16 +800,12 @@ class MergeRequest < ActiveRecord::Base end def fetch_ref - target_project.repository.fetch_ref( - source_project.repository.path_to_repo, - "refs/heads/#{source_branch}", - ref_path - ) + write_ref update_column(:ref_fetched, true) end def ref_path - "refs/merge-requests/#{iid}/head" + "refs/#{Repository::REF_MERGE_REQUEST}/#{iid}/head" end def ref_fetched? @@ -823,7 +827,7 @@ class MergeRequest < ActiveRecord::Base lock_mr yield ensure - unlock_mr if locked? + unlock_mr end end @@ -914,6 +918,12 @@ class MergeRequest < ActiveRecord::Base active_diff_discussions.each do |discussion| service.execute(discussion) end + + if project.resolve_outdated_diff_discussions? + MergeRequests::ResolvedDiscussionNotificationService + .new(project, current_user) + .execute(self) + end end def keep_around_commit @@ -939,4 +949,26 @@ class MergeRequest < ActiveRecord::Base true end + + def update_project_counter_caches? + state_changed? + end + + def update_project_counter_caches + return unless update_project_counter_caches? + + Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache + end + + def first_contribution? + return false if project.team.max_member_access(author_id) > Gitlab::Access::GUEST + + project.merge_requests.merged.where(author_id: author_id).empty? + end + + private + + def write_ref + target_project.repository.fetch_source_branch(source_project.repository, source_branch, ref_path) + end end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index d9d746ccf41..58050e1f438 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -282,9 +282,7 @@ class MergeRequestDiff < ActiveRecord::Base def load_commits commits = st_commits.presence || merge_request_diff_commits - commits.map do |commit| - Commit.new(Gitlab::Git::Commit.new(commit.to_hash), merge_request.source_project) - end + commits.map { |commit| Commit.from_hash(commit.to_hash, project) } end def save_diffs diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index cafdbe11849..670b26d4ca3 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -26,7 +26,7 @@ class MergeRequestDiffCommit < ActiveRecord::Base def to_hash Gitlab::Git::Commit::SERIALIZE_KEYS.each_with_object({}) do |key, hash| - hash[key] = public_send(key) + hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 01e0d0155a3..a3070a12b7c 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -163,7 +163,7 @@ class Milestone < ActiveRecord::Base # Milestone.first.to_reference(same_namespace_project) # => "gitlab-ce%1" # def to_reference(from_project = nil, format: :iid, full: false) - return if is_group_milestone? && format != :name + return if group_milestone? && format != :name format_reference = milestone_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" @@ -207,11 +207,11 @@ class Milestone < ActiveRecord::Base group || project end - def is_group_milestone? + def group_milestone? group_id.present? end - def is_project_milestone? + def project_milestone? project_id.present? end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 6073fb94a3f..4a9a23fea1f 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -44,6 +44,10 @@ class Namespace < ActiveRecord::Base after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } + before_create :sync_share_with_group_lock_with_parent + before_update :sync_share_with_group_lock_with_parent, if: :parent_changed? + after_update :force_share_with_group_lock_on_descendants, if: -> { share_with_group_lock_changed? && share_with_group_lock? } + # Legacy Storage specific hooks after_update :move_dir, if: :path_changed? @@ -156,6 +160,14 @@ class Namespace < ActiveRecord::Base .base_and_ancestors end + def self_and_ancestors + return self.class.where(id: id) unless parent_id + + Gitlab::GroupHierarchy + .new(self.class.where(id: id)) + .base_and_ancestors + end + # Returns all the descendants of the current namespace. def descendants Gitlab::GroupHierarchy @@ -163,6 +175,12 @@ class Namespace < ActiveRecord::Base .base_and_descendants end + def self_and_descendants + Gitlab::GroupHierarchy + .new(self.class.where(id: id)) + .base_and_descendants + end + def user_ids_for_project_authorizations [owner_id] end @@ -181,6 +199,10 @@ class Namespace < ActiveRecord::Base parent.present? end + def subgroup? + has_parent? + end + def soft_delete_without_removing_associations # We can't use paranoia's `#destroy` since this will hard-delete projects. # Project uses `pending_delete` instead of the acts_as_paranoia gem. @@ -201,4 +223,14 @@ class Namespace < ActiveRecord::Base errors.add(:parent_id, "has too deep level of nesting") end end + + def sync_share_with_group_lock_with_parent + if parent&.share_with_group_lock? + self.share_with_group_lock = true + end + end + + def force_share_with_group_lock_on_descendants + descendants.update_all(share_with_group_lock: true) + end end diff --git a/app/models/network/commit.rb b/app/models/network/commit.rb index 8417f200e36..9357e55b419 100644 --- a/app/models/network/commit.rb +++ b/app/models/network/commit.rb @@ -12,7 +12,7 @@ module Network end def method_missing(m, *args, &block) - @commit.send(m, *args, &block) + @commit.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end def space diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb index 2bc00a082df..3845e485413 100644 --- a/app/models/network/graph.rb +++ b/app/models/network/graph.rb @@ -152,14 +152,14 @@ module Network end def find_free_parent_space(range, space_base, space_step, space_default) - if is_overlap?(range, space_default) + if overlap?(range, space_default) find_free_space(range, space_step, space_base, space_default) else space_default end end - def is_overlap?(range, overlap_space) + def overlap?(range, overlap_space) range.each do |i| if i != range.first && i != range.last && @@ -206,7 +206,7 @@ module Network # Visit branching chains leaves.each do |l| - parents = l.parents(@map).select{|p| p.space.zero?} + parents = l.parents(@map).select {|p| p.space.zero?} parents.each do |p| place_chain(p, l.time) end diff --git a/app/models/note.rb b/app/models/note.rb index d0e3bc0bfed..f44590e2144 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -15,6 +15,16 @@ class Note < ActiveRecord::Base include IgnorableColumn include Editable + module SpecialRole + FIRST_TIME_CONTRIBUTOR = :first_time_contributor + + class << self + def values + constants.map {|const| self.const_get(const)} + end + end + end + ignore_column :original_discussion_id cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true @@ -32,9 +42,12 @@ class Note < ActiveRecord::Base # Banzai::ObjectRenderer attr_accessor :user_visible_reference_count - # Attribute used to store the attributes that have ben changed by quick actions. + # Attribute used to store the attributes that have been changed by quick actions. attr_accessor :commands_changes + # A special role that may be displayed on issuable's discussions + attr_accessor :special_role + default_value_for :system, false attr_mentionable :note, pipeline: :note @@ -77,20 +90,20 @@ class Note < ActiveRecord::Base # Scopes scope :for_commit_id, ->(commit_id) { where(noteable_type: "Commit", commit_id: commit_id) } - scope :system, ->{ where(system: true) } - scope :user, ->{ where(system: false) } - scope :common, ->{ where(noteable_type: ["", nil]) } - scope :fresh, ->{ order(created_at: :asc, id: :asc) } - scope :updated_after, ->(time){ where('updated_at > ?', time) } - scope :inc_author_project, ->{ includes(:project, :author) } - scope :inc_author, ->{ includes(:author) } + scope :system, -> { where(system: true) } + scope :user, -> { where(system: false) } + scope :common, -> { where(noteable_type: ["", nil]) } + scope :fresh, -> { order(created_at: :asc, id: :asc) } + scope :updated_after, ->(time) { where('updated_at > ?', time) } + scope :inc_author_project, -> { includes(:project, :author) } + scope :inc_author, -> { includes(:author) } scope :inc_relations_for_view, -> do includes(:project, :author, :updated_by, :resolved_by, :award_emoji, :system_note_metadata) end - scope :diff_notes, ->{ where(type: %w(LegacyDiffNote DiffNote)) } - scope :new_diff_notes, ->{ where(type: 'DiffNote') } - scope :non_diff_notes, ->{ where(type: ['Note', 'DiscussionNote', nil]) } + scope :diff_notes, -> { where(type: %w(LegacyDiffNote DiffNote)) } + scope :new_diff_notes, -> { where(type: 'DiffNote') } + scope :non_diff_notes, -> { where(type: ['Note', 'DiscussionNote', nil]) } scope :with_associations, -> do # FYI noteable cannot be loaded for LegacyDiffNote for commits @@ -141,6 +154,10 @@ class Note < ActiveRecord::Base .group(:noteable_id) .where(noteable_type: type, noteable_id: ids) end + + def has_special_role?(role, note) + note.special_role == role + end end def cross_reference? @@ -206,6 +223,22 @@ class Note < ActiveRecord::Base super(noteable_type.to_s.classify.constantize.base_class.to_s) end + def special_role=(role) + raise "Role is undefined, #{role} not found in #{SpecialRole.values}" unless SpecialRole.values.include?(role) + + @special_role = role + end + + def has_special_role?(role) + self.class.has_special_role?(role, self) + end + + def specialize_for_first_contribution!(noteable) + return unless noteable.author_id == self.author_id + + self.special_role = Note::SpecialRole::FIRST_TIME_CONTRIBUTOR + end + def editable? !system? end @@ -299,6 +332,17 @@ class Note < ActiveRecord::Base end end + def expire_etag_cache + return unless noteable&.discussions_rendered_on_frontend? + + key = Gitlab::Routing.url_helpers.project_noteable_notes_path( + project, + target_type: noteable_type.underscore, + target_id: noteable_id + ) + Gitlab::EtagCaching::Store.new.touch(key) + end + private def keep_around_commit @@ -326,15 +370,4 @@ class Note < ActiveRecord::Base def set_discussion_id self.discussion_id ||= discussion_class.discussion_id(self) end - - def expire_etag_cache - return unless for_issue? - - key = Gitlab::Routing.url_helpers.project_noteable_notes_path( - noteable.project, - target_type: noteable_type.underscore, - target_id: noteable.id - ) - Gitlab::EtagCaching::Store.new.touch(key) - end end diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index 418b42d8f1d..183e098d819 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -5,60 +5,67 @@ class NotificationRecipient custom_action: nil, target: nil, acting_user: nil, - project: nil + project: nil, + group: nil, + skip_read_ability: false ) + unless NotificationSetting.levels.key?(type) || type == :subscription + raise ArgumentError, "invalid type: #{type.inspect}" + end + @custom_action = custom_action @acting_user = acting_user @target = target - @project = project || @target&.project + @project = project || default_project + @group = group || @project&.group @user = user @type = type + @skip_read_ability = skip_read_ability end def notification_setting @notification_setting ||= find_notification_setting end - def raw_notification_level - notification_setting&.level&.to_sym - end - def notification_level - # custom is treated the same as watch if it's enabled - otherwise it's - # set to :custom, meaning to send exactly when our type is :participating - # or :mention. - @notification_level ||= - case raw_notification_level - when :custom - if @custom_action && notification_setting&.event_enabled?(@custom_action) - :watch - else - :custom - end - else - raw_notification_level - end + @notification_level ||= notification_setting&.level&.to_sym end def notifiable? return false unless has_access? return false if own_activity? - return true if @type == :subscription - - return false if notification_level.nil? || notification_level == :disabled - - return %i[participating mention].include?(@type) if notification_level == :custom - - return false if %i[watch participating].include?(notification_level) && excluded_watcher_action? + # even users with :disabled notifications receive manual subscriptions + return !unsubscribed? if @type == :subscription - return false unless NotificationSetting.levels[notification_level] <= NotificationSetting.levels[@type] + return false unless suitable_notification_level? + # check this last because it's expensive + # nobody should receive notifications if they've specifically unsubscribed return false if unsubscribed? true end + def suitable_notification_level? + case notification_level + when :disabled, nil + false + when :custom + custom_enabled? || %i[participating mention].include?(@type) + when :watch, :participating + !excluded_watcher_action? + when :mention + @type == :mention + else + false + end + end + + def custom_enabled? + @custom_action && notification_setting&.event_enabled?(@custom_action) + end + def unsubscribed? return false unless @target return false unless @target.respond_to?(:subscriptions) @@ -77,6 +84,8 @@ class NotificationRecipient def has_access? DeclarativePolicy.subject_scope do return false unless user.can?(:receive_notifications) + return true if @skip_read_ability + return false if @project && !user.can?(:read_project, @project) return true unless read_ability @@ -88,7 +97,7 @@ class NotificationRecipient def excluded_watcher_action? return false unless @custom_action - return false if raw_notification_level == :custom + return false if notification_level == :custom NotificationSetting::EXCLUDED_WATCHER_EVENTS.include?(@custom_action) end @@ -96,6 +105,7 @@ class NotificationRecipient private def read_ability + return nil if @skip_read_ability return @read_ability if instance_variable_defined?(:@read_ability) @read_ability = @@ -111,12 +121,18 @@ class NotificationRecipient end end + def default_project + return nil if @target.nil? + return @target if @target.is_a?(Project) + return @target.project if @target.respond_to?(:project) + end + def find_notification_setting project_setting = @project && user.notification_settings_for(@project) return project_setting unless project_setting.nil? || project_setting.global? - group_setting = @project&.group && user.notification_settings_for(@project.group) + group_setting = @group && user.notification_settings_for(@group) return group_setting unless group_setting.nil? || group_setting.global? diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 9b1cac64c44..245f8dddcf9 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -66,6 +66,6 @@ class NotificationSetting < ActiveRecord::Base alias_method :failed_pipeline?, :failed_pipeline def event_enabled?(event) - respond_to?(event) && !!public_send(event) + respond_to?(event) && !!public_send(event) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/models/project.rb b/app/models/project.rb index e7baba2ef08..ff5638dd155 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -17,14 +17,15 @@ class Project < ActiveRecord::Base include ProjectFeaturesCompatibility include SelectForProjectAuthorization include Routable - include Storage::LegacyProject extend Gitlab::ConfigHelper + extend Gitlab::CurrentSettings BoardLimitExceeded = Class.new(StandardError) NUMBER_OF_PERMITTED_BOARDS = 1 UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze + LATEST_STORAGE_VERSION = 1 cache_markdown_field :description, pipeline: :description @@ -32,8 +33,11 @@ class Project < ActiveRecord::Base :merge_requests_enabled?, :issues_enabled?, to: :project_feature, allow_nil: true + delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage + default_value_for :archived, false default_value_for :visibility_level, gitlab_config_features.visibility_level + default_value_for :resolve_outdated_diff_discussions, false default_value_for :container_registry_enabled, gitlab_config_features.container_registry default_value_for(:repository_storage) { current_application_settings.pick_repository_storage } default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } @@ -44,36 +48,27 @@ class Project < ActiveRecord::Base default_value_for :snippets_enabled, gitlab_config_features.snippets default_value_for :only_allow_merge_if_all_discussions_are_resolved, false - after_create :ensure_storage_path_exist - after_create :create_project_feature, unless: :project_feature - after_save :update_project_statistics, if: :namespace_id_changed? + add_authentication_token_field :runners_token + before_save :ensure_runners_token - # set last_activity_at to the same as created_at + after_save :update_project_statistics, if: :namespace_id_changed? + after_create :create_project_feature, unless: :project_feature after_create :set_last_activity_at - def set_last_activity_at - update_column(:last_activity_at, self.created_at) - end - after_create :set_last_repository_updated_at - def set_last_repository_updated_at - update_column(:last_repository_updated_at, self.created_at) - end + after_update :update_forks_visibility_level before_destroy :remove_private_deploy_keys - after_destroy :remove_pages - - # update visibility_level of forks - after_update :update_forks_visibility_level + after_destroy -> { run_after_commit { remove_pages } } after_validation :check_pending_delete - # Legacy Storage specific hooks - - after_save :ensure_storage_path_exist, if: :namespace_id_changed? + # Storage specific hooks + after_initialize :use_hashed_storage + after_create :ensure_storage_path_exists + after_save :ensure_storage_path_exists, if: :namespace_id_changed? acts_as_taggable - attr_accessor :new_default_branch attr_accessor :old_path_with_namespace attr_accessor :template_name attr_writer :pipeline_status @@ -150,6 +145,7 @@ class Project < ActiveRecord::Base has_many :requesters, -> { where.not(requested_at: nil) }, as: :source, class_name: 'ProjectMember', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_many :members_and_requesters, as: :source, class_name: 'ProjectMember' has_many :deploy_keys_projects has_many :deploy_keys, through: :deploy_keys_projects @@ -191,12 +187,14 @@ class Project < ActiveRecord::Base has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_one :auto_devops, class_name: 'ProjectAutoDevops' + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature accepts_nested_attributes_for :import_data + accepts_nested_attributes_for :auto_devops delegate :name, to: :owner, allow_nil: true, prefix: true - delegate :count, to: :forks, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team @@ -229,6 +227,7 @@ class Project < ActiveRecord::Base validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?] validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create + validate :can_create_repository?, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? } validate :avatar_type, if: ->(project) { project.avatar.present? && project.avatar_changed? } validates :avatar, file_size: { maximum: 200.kilobytes.to_i } @@ -239,9 +238,6 @@ class Project < ActiveRecord::Base presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } - add_authentication_token_field :runners_token - before_save :ensure_runners_token - mount_uploader :avatar, AvatarUploader has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -257,6 +253,7 @@ class Project < ActiveRecord::Base scope :joined, ->(user) { where('namespace_id != ?', user.namespace_id) } scope :starred_by, ->(user) { joins(:users_star_projects).where('users_star_projects.user_id': user.id) } scope :visible_to_user, ->(user) { where(id: user.authorized_projects.select(:id).reorder(nil)) } + scope :archived, -> { where(archived: true) } scope :non_archived, -> { where(archived: false) } scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } @@ -370,7 +367,10 @@ class Project < ActiveRecord::Base state :failed after_transition [:none, :finished, :failed] => :scheduled do |project, _| - project.run_after_commit { add_import_job } + project.run_after_commit do + job_id = add_import_job + update(import_jid: job_id) if job_id + end end after_transition started: :finished do |project, _| @@ -378,11 +378,7 @@ class Project < ActiveRecord::Base if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists? project.run_after_commit do - begin - Projects::HousekeepingService.new(project).execute - rescue Projects::HousekeepingService::LeaseTaken => e - Rails.logger.info("Could not perform housekeeping for project #{project.full_path} (#{project.id}): #{e}") - end + Projects::AfterImportService.new(project).execute end end end @@ -415,7 +411,7 @@ class Project < ActiveRecord::Base union = Gitlab::SQL::Union.new([projects, namespaces]) - where("projects.id IN (#{union.to_sql})") + where("projects.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end def search_by_title(query) @@ -473,8 +469,20 @@ class Project < ActiveRecord::Base self[:lfs_enabled] && Gitlab.config.lfs.enabled end + def auto_devops_enabled? + if auto_devops&.enabled.nil? + current_application_settings.auto_devops_enabled? + else + auto_devops.enabled? + end + end + + def has_auto_devops_implicitly_disabled? + auto_devops&.enabled.nil? && !current_application_settings.auto_devops_enabled? + end + def repository_storage_path - Gitlab.config.repositories.storages[repository_storage]['path'] + Gitlab.config.repositories.storages[repository_storage].try(:[], 'path') end def team @@ -485,6 +493,10 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(full_path, self, disk_path: disk_path) end + def reload_repository! + @repository = nil + end + def container_registry_url if Gitlab.config.registry.enabled "#{Gitlab.config.registry.host_port}/#{full_path.downcase}" @@ -525,17 +537,26 @@ class Project < ActiveRecord::Base def add_import_job job_id = if forked? - RepositoryForkWorker.perform_async(id, forked_from_project.repository_storage_path, - forked_from_project.full_path, - self.namespace.full_path) + RepositoryForkWorker.perform_async(id, + forked_from_project.repository_storage_path, + forked_from_project.full_path, + self.namespace.full_path) else RepositoryImportWorker.perform_async(self.id) end + log_import_activity(job_id) + + job_id + end + + def log_import_activity(job_id, type: :import) + job_type = type.to_s.capitalize + if job_id - Rails.logger.info "Import job started for #{full_path} with job ID #{job_id}" + Rails.logger.info("#{job_type} job scheduled for #{full_path} with job ID #{job_id}.") else - Rails.logger.error "Import job failed to start for #{full_path}" + Rails.logger.error("#{job_type} job failed to create for #{full_path}.") end end @@ -544,6 +565,7 @@ class Project < ActiveRecord::Base ProjectCacheWorker.perform_async(self.id) end + update(import_error: nil) remove_import_data end @@ -575,7 +597,7 @@ class Project < ActiveRecord::Base end def valid_import_url? - valid? || errors.messages[:import_url].nil? + valid?(:import_url) || errors.messages[:import_url].nil? end def create_or_update_import_data(data: nil, credentials: nil) @@ -825,7 +847,7 @@ class Project < ActiveRecord::Base if template.nil? # If no template, we should create an instance. Ex `build_gitlab_ci_service` - public_send("build_#{service_name}_service") + public_send("build_#{service_name}_service") # rubocop:disable GitlabSecurity/PublicSend else Service.build_from_template(id, template) end @@ -921,14 +943,14 @@ class Project < ActiveRecord::Base end def execute_hooks(data, hooks_scope = :push_hooks) - hooks.send(hooks_scope).each do |hook| + hooks.public_send(hooks_scope).each do |hook| # rubocop:disable GitlabSecurity/PublicSend hook.async_execute(data, hooks_scope.to_s) end end def execute_services(data, hooks_scope = :push_hooks) # Call only service hooks that are active for this scope - services.send(hooks_scope).each do |service| + services.public_send(hooks_scope).each do |service| # rubocop:disable GitlabSecurity/PublicSend service.async_execute(data) end end @@ -992,12 +1014,39 @@ class Project < ActiveRecord::Base end end + # Check if repository already exists on disk + def can_create_repository? + return false unless repository_storage_path + + expires_full_path_cache # we need to clear cache to validate renames correctly + + if gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git") + errors.add(:base, 'There is already a repository with that name on disk') + return false + end + + true + end + + def create_repository(force: false) + # Forked import is handled asynchronously + return if forked? && !force + + if gitlab_shell.add_repository(repository_storage_path, disk_path) + repository.after_create + true + else + errors.add(:base, 'Failed to create repository via gitlab-shell') + false + end + end + def hook_attrs(backward: true) attrs = { name: name, description: description, web_url: web_url, - avatar_url: avatar_url, + avatar_url: avatar_url(only_path: false), git_ssh_url: ssh_url_to_repo, git_http_url: http_url_to_repo, namespace: namespace.name, @@ -1046,13 +1095,16 @@ class Project < ActiveRecord::Base end def change_head(branch) - repository.before_change_head - repository.rugged.references.create('HEAD', - "refs/heads/#{branch}", - force: true) - repository.copy_gitattributes(branch) - repository.after_change_head - reload_default_branch + if repository.branch_exists?(branch) + repository.before_change_head + repository.write_ref('HEAD', "refs/heads/#{branch}") + repository.copy_gitattributes(branch) + repository.after_change_head + reload_default_branch + else + errors.add(:base, "Could not change HEAD: branch '#{branch}' does not exist") + false + end end def forked_from?(project) @@ -1071,6 +1123,7 @@ class Project < ActiveRecord::Base !!repository.exists? end + # update visibility_level of forks def update_forks_visibility_level return unless visibility_level < visibility_level_was @@ -1143,7 +1196,11 @@ class Project < ActiveRecord::Base end def open_issues_count - issues.opened.count + Projects::OpenIssuesCountService.new(self).count + end + + def open_merge_requests_count + Projects::OpenMergeRequestsCountService.new(self).count end def visibility_level_allowed_as_fork?(level = self.visibility_level) @@ -1198,13 +1255,18 @@ class Project < ActiveRecord::Base end def pages_path - File.join(Settings.pages.path, disk_path) + # TODO: when we migrate Pages to work with new storage types, change here to use disk_path + File.join(Settings.pages.path, full_path) end def public_pages_path File.join(pages_path, 'public') end + def pages_available? + Gitlab.config.pages.enabled && !namespace.subgroup? + end + def remove_private_deploy_keys exclude_keys_linked_to_other_projects = <<-SQL NOT EXISTS ( @@ -1222,6 +1284,9 @@ class Project < ActiveRecord::Base # TODO: what to do here when not using Legacy Storage? Do we still need to rename and delay removal? def remove_pages + # Projects with a missing namespace cannot have their pages removed + return unless namespace + ::Projects::UpdatePagesConfigurationService.new(self).execute # 1. We rename pages to temporary directory @@ -1234,6 +1299,50 @@ class Project < ActiveRecord::Base end end + def rename_repo + new_full_path = build_full_path + + Rails.logger.error "Attempting to rename #{full_path_was} -> #{new_full_path}" + + if has_container_registry_tags? + Rails.logger.error "Project #{full_path_was} cannot be renamed because container registry tags are present!" + + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') + end + + expire_caches_before_rename(full_path_was) + + if storage.rename_repo + Gitlab::AppLogger.info "Project was renamed: #{full_path_was} -> #{new_full_path}" + rename_repo_notify! + after_rename_repo + else + Rails.logger.error "Repository could not be renamed: #{full_path_was} -> #{new_full_path}" + + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise StandardError.new('repository cannot be renamed') + end + end + + def rename_repo_notify! + send_move_instructions(full_path_was) + expires_full_path_cache + + self.old_path_with_namespace = full_path_was + SystemHooksService.new.execute_hooks_for(self, :rename) + + reload_repository! + end + + def after_rename_repo + path_before_change = previous_changes['path'].first + + Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) + Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) + end + def running_or_pending_build_count(force: false) Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do builds.running_or_pending.count(:all) @@ -1280,12 +1389,20 @@ class Project < ActiveRecord::Base status.zero? end + def full_path_slug + Gitlab::Utils.slugify(full_path.to_s) + end + + def has_ci? + repository.gitlab_ci_yml || auto_devops_enabled? + end + def predefined_variables [ { key: 'CI_PROJECT_ID', value: id.to_s, public: true }, { key: 'CI_PROJECT_NAME', value: path, public: true }, { key: 'CI_PROJECT_PATH', value: full_path, public: true }, - { key: 'CI_PROJECT_PATH_SLUG', value: full_path.parameterize, public: true }, + { key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug, public: true }, { key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path, public: true }, { key: 'CI_PROJECT_URL', value: web_url, public: true } ] @@ -1325,8 +1442,14 @@ class Project < ActiveRecord::Base deployment_service.predefined_variables end + def auto_devops_variables + return [] unless auto_devops_enabled? + + auto_devops&.variables || [] + end + def append_or_update_attribute(name, value) - old_values = public_send(name.to_s) + old_values = public_send(name.to_s) # rubocop:disable GitlabSecurity/PublicSend if Project.reflect_on_association(name).try(:macro) == :has_many && old_values.any? update_attribute(name, old_values + value) @@ -1388,13 +1511,61 @@ class Project < ActiveRecord::Base end end + def multiple_issue_boards_available?(user) + feature_available?(:multiple_issue_boards, user) + end + + def issue_board_milestone_available?(user = nil) + feature_available?(:issue_board_milestone, user) + end + + def full_path_was + File.join(namespace.full_path, previous_changes['path'].first) + end + alias_method :name_with_namespace, :full_name alias_method :human_name, :full_name # @deprecated cannot remove yet because it has an index with its name in elasticsearch alias_method :path_with_namespace, :full_path + def forks_count + Projects::ForksCountService.new(self).count + end + + def legacy_storage? + self.storage_version.nil? + end + + def renamed? + persisted? && path_changed? + end + private + def storage + @storage ||= + if self.storage_version && self.storage_version >= 1 + Storage::HashedProject.new(self) + else + Storage::LegacyProject.new(self) + end + end + + def use_hashed_storage + if self.new_record? && current_application_settings.hashed_storage_enabled + self.storage_version = LATEST_STORAGE_VERSION + end + end + + # set last_activity_at to the same as created_at + def set_last_activity_at + update_column(:last_activity_at, self.created_at) + end + + def set_last_repository_updated_at + update_column(:last_repository_updated_at, self.created_at) + end + def cross_namespace_reference?(from) case from when Project diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb new file mode 100644 index 00000000000..53731579e87 --- /dev/null +++ b/app/models/project_auto_devops.rb @@ -0,0 +1,11 @@ +class ProjectAutoDevops < ActiveRecord::Base + belongs_to :project + + validates :domain, allow_blank: true, hostname: { allow_numeric_hostname: true } + + def variables + variables = [] + variables << { key: 'AUTO_DEVOPS_DOMAIN', value: domain, public: true } if domain.present? + variables + end +end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index c8fabb16dc1..fb1db0255aa 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -55,7 +55,7 @@ class ProjectFeature < ActiveRecord::Base end def access_level(feature) - public_send(ProjectFeature.access_level_attribute(feature)) + public_send(ProjectFeature.access_level_attribute(feature)) # rubocop:disable GitlabSecurity/PublicSend end def builds_enabled? @@ -80,7 +80,7 @@ class ProjectFeature < ActiveRecord::Base # which cannot be higher than repository access level def repository_children_level validator = lambda do |field| - level = public_send(field) || ProjectFeature::ENABLED + level = public_send(field) || ProjectFeature::ENABLED # rubocop:disable GitlabSecurity/PublicSend not_allowed = level > repository_access_level self.errors.add(field, "cannot have higher visibility level than repository access level") if not_allowed end diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 6d1a321f651..818cfb01b14 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -101,9 +101,9 @@ class ChatNotificationService < Service when "push", "tag_push" ChatMessage::PushMessage.new(data) when "issue" - ChatMessage::IssueMessage.new(data) unless is_update?(data) + ChatMessage::IssueMessage.new(data) unless update?(data) when "merge_request" - ChatMessage::MergeMessage.new(data) unless is_update?(data) + ChatMessage::MergeMessage.new(data) unless update?(data) when "note" ChatMessage::NoteMessage.new(data) when "pipeline" @@ -115,7 +115,7 @@ class ChatNotificationService < Service def get_channel_field(event) field_name = event_channel_name(event) - self.public_send(field_name) + self.public_send(field_name) # rubocop:disable GitlabSecurity/PublicSend end def build_event_channels @@ -136,7 +136,7 @@ class ChatNotificationService < Service project.web_url end - def is_update?(data) + def update?(data) data[:object_attributes][:action] == 'update' end diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index e3906943ecd..976d85246a8 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -53,7 +53,7 @@ class HipchatService < Service return unless supported_events.include?(data[:object_kind]) message = create_message(data) return unless message.present? - gate[room].send('GitLab', message, message_options(data)) + gate[room].send('GitLab', message, message_options(data)) # rubocop:disable GitlabSecurity/PublicSend end def test(data) @@ -85,9 +85,9 @@ class HipchatService < Service when "push", "tag_push" create_push_message(data) when "issue" - create_issue_message(data) unless is_update?(data) + create_issue_message(data) unless update?(data) when "merge_request" - create_merge_request_message(data) unless is_update?(data) + create_merge_request_message(data) unless update?(data) when "note" create_note_message(data) when "pipeline" @@ -282,7 +282,7 @@ class HipchatService < Service "<a href=\"#{project_url}\">#{project_name}</a>" end - def is_update?(data) + def update?(data) data[:object_attributes][:action] == 'update' end diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index dee99bbb859..8ba07173c74 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -24,6 +24,8 @@ class KubernetesService < DeploymentService validates :token end + before_validation :enforce_namespace_to_lower_case + validates :namespace, allow_blank: true, length: 1..63, @@ -207,4 +209,8 @@ class KubernetesService < DeploymentService max_session_time: current_application_settings.terminal_max_session_time } end + + def enforce_namespace_to_lower_case + self.namespace = self.namespace&.downcase + end end diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index aeaf63abab9..715b215d1db 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -14,7 +14,7 @@ class ProjectStatistics < ActiveRecord::Base def refresh!(only: nil) STATISTICS_COLUMNS.each do |column, generator| if only.blank? || only.include?(column) - public_send("update_#{column}") + public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 674eacd28e8..09049824ff7 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -150,7 +150,7 @@ class ProjectTeam end def human_max_access(user_id) - Gitlab::Access.options_with_owner.key(max_member_access(user_id)) + Gitlab::Access.human_access(max_member_access(user_id)) end # Determine the maximum access level for a group of users in bulk. diff --git a/app/models/protectable_dropdown.rb b/app/models/protectable_dropdown.rb index 122fbce257d..c96edc5a259 100644 --- a/app/models/protectable_dropdown.rb +++ b/app/models/protectable_dropdown.rb @@ -1,5 +1,9 @@ class ProtectableDropdown + REF_TYPES = %i[branches tags].freeze + def initialize(project, ref_type) + raise ArgumentError, "invalid ref type `#{ref_type}`" unless ref_type.in?(REF_TYPES) + @project = project @ref_type = ref_type end @@ -16,7 +20,7 @@ class ProtectableDropdown private def refs - @project.repository.public_send(@ref_type) + @project.repository.public_send(@ref_type) # rubocop:disable GitlabSecurity/PublicSend end def ref_names @@ -24,7 +28,7 @@ class ProtectableDropdown end def protections - @project.public_send("protected_#{@ref_type}") + @project.public_send("protected_#{@ref_type}") # rubocop:disable GitlabSecurity/PublicSend end def non_wildcard_protected_ref_names diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 5f0d0802ac9..89bfc5f9a9c 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -2,6 +2,8 @@ class ProtectedBranch < ActiveRecord::Base include Gitlab::ShellAdapter include ProtectedRef + extend Gitlab::CurrentSettings + protected_ref_access_levels :merge, :push # Check if branch name is marked as protected in the system diff --git a/app/models/push_event.rb b/app/models/push_event.rb new file mode 100644 index 00000000000..23ffb0d4ea8 --- /dev/null +++ b/app/models/push_event.rb @@ -0,0 +1,81 @@ +class PushEvent < Event + # This validation exists so we can't accidentally use PushEvent with a + # different "action" value. + validate :validate_push_action + + # Authors are required as they're used to display who pushed data. + # + # We're just validating the presence of the ID here as foreign key constraints + # should ensure the ID points to a valid user. + validates :author_id, presence: true + + # The project is required to build links to commits, commit ranges, etc. + # + # We're just validating the presence of the ID here as foreign key constraints + # should ensure the ID points to a valid project. + validates :project_id, presence: true + + # These fields are also not used for push events, thus storing them would be a + # waste. + validates :target_id, absence: true + validates :target_type, absence: true + + delegate :branch?, to: :push_event_payload + delegate :tag?, to: :push_event_payload + delegate :commit_from, to: :push_event_payload + delegate :commit_to, to: :push_event_payload + delegate :ref_type, to: :push_event_payload + delegate :commit_title, to: :push_event_payload + + delegate :commit_count, to: :push_event_payload + alias_method :commits_count, :commit_count + + def self.sti_name + PUSHED + end + + def push? + true + end + + def push_with_commits? + !!(commit_from && commit_to) + end + + def valid_push? + push_event_payload.ref.present? + end + + def new_ref? + push_event_payload.created? + end + + def rm_ref? + push_event_payload.removed? + end + + def md_ref? + !(rm_ref? || new_ref?) + end + + def ref_name + push_event_payload.ref + end + + alias_method :branch_name, :ref_name + alias_method :tag_name, :ref_name + + def commit_id + commit_to || commit_from + end + + def last_push_to_non_root? + branch? && project.default_branch != branch_name + end + + def validate_push_action + return if action == PUSHED + + errors.add(:action, "the action #{action.inspect} is not valid") + end +end diff --git a/app/models/push_event_payload.rb b/app/models/push_event_payload.rb new file mode 100644 index 00000000000..6cdb1cd4fe9 --- /dev/null +++ b/app/models/push_event_payload.rb @@ -0,0 +1,22 @@ +class PushEventPayload < ActiveRecord::Base + include ShaAttribute + + belongs_to :event, inverse_of: :push_event_payload + + validates :event_id, :commit_count, :action, :ref_type, presence: true + validates :commit_title, length: { maximum: 70 } + + sha_attribute :commit_from + sha_attribute :commit_to + + enum action: { + created: 0, + removed: 1, + pushed: 2 + } + + enum ref_type: { + branch: 0, + tag: 1 + } +end diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb index 964175ddab8..31de204d824 100644 --- a/app/models/redirect_route.rb +++ b/app/models/redirect_route.rb @@ -8,5 +8,13 @@ class RedirectRoute < ActiveRecord::Base presence: true, uniqueness: { case_sensitive: false } - scope :matching_path_and_descendants, -> (path) { where('redirect_routes.path = ? OR redirect_routes.path LIKE ?', path, "#{sanitize_sql_like(path)}/%") } + scope :matching_path_and_descendants, -> (path) do + wheres = if Gitlab::Database.postgresql? + 'LOWER(redirect_routes.path) = LOWER(?) OR LOWER(redirect_routes.path) LIKE LOWER(?)' + else + 'redirect_routes.path = ? OR redirect_routes.path LIKE ?' + end + + where(wheres, path, "#{sanitize_sql_like(path)}/%") + end end diff --git a/app/models/repository.rb b/app/models/repository.rb index 3b5d0e00c70..035f85a0b46 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1,6 +1,18 @@ require 'securerandom' class Repository + REF_MERGE_REQUEST = 'merge-requests'.freeze + REF_KEEP_AROUND = 'keep-around'.freeze + REF_ENVIRONMENTS = 'environments'.freeze + + RESERVED_REFS_NAMES = %W[ + heads + tags + #{REF_ENVIRONMENTS} + #{REF_KEEP_AROUND} + #{REF_ENVIRONMENTS} + ].freeze + include Gitlab::ShellAdapter include RepositoryMirroring @@ -8,7 +20,6 @@ class Repository delegate :ref_name_for_sha, to: :raw_repository - CommitError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) # Methods that cache data from the Git repository. @@ -48,7 +59,9 @@ class Repository alias_method(original, name) define_method(name) do - cache_method_output(name, fallback: fallback, memoize_only: memoize_only) { __send__(original) } + cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do + __send__(original) # rubocop:disable GitlabSecurity/PublicSend + end end end @@ -58,12 +71,18 @@ class Repository @project = project end + def ==(other) + @disk_path == other.disk_path + end + def raw_repository return nil unless full_path @raw_repository ||= initialize_raw_repository end + alias_method :raw, :raw_repository + # Return absolute path to repository def path_to_repo @path_to_repo ||= File.expand_path( @@ -71,17 +90,8 @@ class Repository ) end - # - # Git repository can contains some hidden refs like: - # /refs/notes/* - # /refs/git-as-svn/* - # /refs/pulls/* - # This refs by default not visible in project page and not cloned to client side. - # - # This method return true if repository contains some content visible in project page. - # - def has_visible_content? - branch_count > 0 + def inspect + "#<#{self.class.name}:#{@disk_path}>" end def commit(ref = 'HEAD') @@ -156,32 +166,25 @@ class Repository end def add_branch(user, branch_name, ref) - newrev = commit(ref).try(:sha) - - return false unless newrev - - GitOperationService.new(user, self).add_branch(branch_name, newrev) + branch = raw_repository.add_branch(branch_name, committer: user, target: ref) after_create_branch - find_branch(branch_name) + + branch + rescue Gitlab::Git::Repository::InvalidRef + false end def add_tag(user, tag_name, target, message = nil) - newrev = commit(target).try(:id) - options = { message: message, tagger: user_to_committer(user) } if message - - return false unless newrev - - GitOperationService.new(user, self).add_tag(tag_name, newrev, options) - - find_tag(tag_name) + raw_repository.add_tag(tag_name, committer: user, target: target, message: message) + rescue Gitlab::Git::Repository::InvalidRef + false end def rm_branch(user, branch_name) before_remove_branch - branch = find_branch(branch_name) - GitOperationService.new(user, self).rm_branch(branch) + raw_repository.rm_branch(branch_name, committer: user) after_remove_branch true @@ -189,9 +192,8 @@ class Repository def rm_tag(user, tag_name) before_remove_tag - tag = find_tag(tag_name) - GitOperationService.new(user, self).rm_tag(tag) + raw_repository.rm_tag(tag_name, committer: user) after_remove_tag true @@ -202,12 +204,18 @@ class Repository end def branch_exists?(branch_name) - branch_names.include?(branch_name) + return false unless raw_repository + + @branch_exists_memo ||= Hash.new do |hash, key| + hash[key] = raw_repository.branch_exists?(key) + end + + @branch_exists_memo[branch_name] end def ref_exists?(ref) - rugged.references.exist?(ref) - rescue Rugged::ReferenceError + !!raw_repository&.ref_exists?(ref) + rescue ArgumentError false end @@ -222,12 +230,12 @@ class Repository # This will still fail if the file is corrupted (e.g. 0 bytes) begin - rugged.references.create(keep_around_ref_name(sha), sha, force: true) + write_ref(keep_around_ref_name(sha), sha) rescue Rugged::ReferenceError => ex - Rails.logger.error "Unable to create keep-around reference for repository #{path}: #{ex}" + Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" rescue Rugged::OSError => ex raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ - Rails.logger.error "Unable to create keep-around reference for repository #{path}: #{ex}" + Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" end end @@ -235,6 +243,10 @@ class Repository ref_exists?(keep_around_ref_name(sha)) end + def write_ref(ref_path, sha) + rugged.references.create(ref_path, sha, force: true) + end + def diverging_commit_counts(branch) root_ref_hash = raw_repository.rev_parse_target(root_ref).oid cache.fetch(:"diverging_commit_counts_#{branch.name}") do @@ -258,6 +270,7 @@ class Repository def expire_branches_cache expire_method_caches(%i(branch_names branch_count)) @local_branches = nil + @branch_exists_memo = nil end def expire_statistics_caches @@ -298,7 +311,7 @@ class Repository expire_method_caches(to_refresh) - to_refresh.each { |method| send(method) } + to_refresh.each { |method| send(method) } # rubocop:disable GitlabSecurity/PublicSend end def expire_branch_cache(branch_name = nil) @@ -437,9 +450,9 @@ class Repository def method_missing(m, *args, &block) if m == :lookup && !block_given? lookup_cache[m] ||= {} - lookup_cache[m][args.join(":")] ||= raw_repository.send(m, *args, &block) + lookup_cache[m][args.join(":")] ||= raw_repository.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend else - raw_repository.send(m, *args, &block) + raw_repository.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end end @@ -749,28 +762,42 @@ class Repository multi_action(**options) end + def with_branch(user, *args) + result = Gitlab::Git::OperationService.new(user, raw_repository).with_branch(*args) do |start_commit| + yield start_commit + end + + newrev, should_run_after_create, should_run_after_create_branch = result + + after_create if should_run_after_create + after_create_branch if should_run_after_create_branch + + newrev + end + # rubocop:disable Metrics/ParameterLists def multi_action( user:, branch_name:, message:, actions:, author_email: nil, author_name: nil, start_branch_name: nil, start_project: project) - GitOperationService.new(user, self).with_branch( + with_branch( + user, branch_name, start_branch_name: start_branch_name, - start_project: start_project) do |start_commit| + start_repository: start_project.repository.raw_repository) do |start_commit| index = Gitlab::Git::Index.new(raw_repository) if start_commit - index.read_tree(start_commit.raw_commit.tree) + index.read_tree(start_commit.rugged_commit.tree) parents = [start_commit.sha] else parents = [] end actions.each do |options| - index.public_send(options.delete(:action), options) + index.public_send(options.delete(:action), options) # rubocop:disable GitlabSecurity/PublicSend end options = { @@ -811,7 +838,8 @@ class Repository end def merge(user, source, merge_request, options = {}) - GitOperationService.new(user, self).with_branch( + with_branch( + user, merge_request.target_branch) do |start_commit| our_commit = start_commit.sha their_commit = source @@ -831,17 +859,18 @@ class Repository merge_request.update(in_progress_merge_commit_sha: commit_id) commit_id end - rescue Repository::CommitError # when merge_index.conflicts? + rescue Gitlab::Git::CommitError # when merge_index.conflicts? false end def revert( user, commit, branch_name, start_branch_name: nil, start_project: project) - GitOperationService.new(user, self).with_branch( + with_branch( + user, branch_name, start_branch_name: start_branch_name, - start_project: start_project) do |start_commit| + start_repository: start_project.repository.raw_repository) do |start_commit| revert_tree_id = check_revert_content(commit, start_commit.sha) unless revert_tree_id @@ -861,10 +890,11 @@ class Repository def cherry_pick( user, commit, branch_name, start_branch_name: nil, start_project: project) - GitOperationService.new(user, self).with_branch( + with_branch( + user, branch_name, start_branch_name: start_branch_name, - start_project: start_project) do |start_commit| + start_repository: start_project.repository.raw_repository) do |start_commit| cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) unless cherry_pick_tree_id @@ -873,7 +903,7 @@ class Repository committer = user_to_committer(user) - create_commit(message: commit.message, + create_commit(message: commit.cherry_pick_message(user), author: { email: commit.author_email, name: commit.author_name, @@ -886,7 +916,7 @@ class Repository end def resolve_conflicts(user, branch_name, params) - GitOperationService.new(user, self).with_branch(branch_name) do + with_branch(user, branch_name) do committer = user_to_committer(user) create_commit(params.merge(author: committer, committer: committer)) @@ -929,7 +959,7 @@ class Repository if branch_commit same_head = branch_commit.id == root_ref_commit.id - !same_head && is_ancestor?(branch_commit.id, root_ref_commit.id) + !same_head && ancestor?(branch_commit.id, root_ref_commit.id) else nil end @@ -943,12 +973,12 @@ class Repository nil end - def is_ancestor?(ancestor_id, descendant_id) + def ancestor?(ancestor_id, descendant_id) return false if ancestor_id.nil? || descendant_id.nil? Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| if is_enabled - raw_repository.is_ancestor?(ancestor_id, descendant_id) + raw_repository.ancestor?(ancestor_id, descendant_id) else rugged_is_ancestor?(ancestor_id, descendant_id) end @@ -976,30 +1006,6 @@ class Repository run_git(args).first.lines.map(&:strip) end - def with_repo_branch_commit(start_repository, start_branch_name) - return yield(nil) if start_repository.empty_repo? - - branch_name_or_sha = - if start_repository == self - start_branch_name - else - tmp_ref = "refs/tmp/#{SecureRandom.hex}/head" - - fetch_ref( - start_repository.path_to_repo, - "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", - tmp_ref - ) - - start_repository.commit(start_branch_name).sha - end - - yield(commit(branch_name_or_sha)) - - ensure - rugged.references.delete(tmp_ref) if tmp_ref - end - def add_remote(name, url) raw_repository.remote_add(name, url) rescue Rugged::ConfigError @@ -1014,12 +1020,15 @@ class Repository end def fetch_remote(remote, forced: false, no_tags: false) - gitlab_shell.fetch_remote(repository_storage_path, disk_path, remote, forced: forced, no_tags: no_tags) + gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags) + end + + def fetch_source_branch(source_repository, source_branch, local_ref) + raw_repository.fetch_source_branch(source_repository.raw_repository, source_branch, local_ref) end - def fetch_ref(source_path, source_ref, target_ref) - args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) - run_git(args) + def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) + raw_repository.compare_source_branch(target_branch_name, source_repository.raw_repository, source_branch_name, straight: straight) end def create_ref(ref, ref_path) @@ -1100,12 +1109,6 @@ class Repository private - def run_git(args) - circuit_breaker.perform do - Gitlab::Popen.popen([Gitlab.config.git.bin_path, *args], path_to_repo) - end - end - def blob_data_at(sha, path) blob = blob_at(sha, path) return unless blob @@ -1141,7 +1144,7 @@ class Repository end def keep_around_ref_name(sha) - "refs/keep-around/#{sha}" + "refs/#{REF_KEEP_AROUND}/#{sha}" end def repository_event(event, tags = {}) @@ -1174,7 +1177,7 @@ class Repository end def initialize_raw_repository - Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git') + Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false)) end def circuit_breaker diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 09d5ff46618..9533aa7f555 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -10,6 +10,8 @@ class Snippet < ActiveRecord::Base include Spammable include Editable + extend Gitlab::CurrentSettings + cache_markdown_field :title, pipeline: :single_line cache_markdown_field :description cache_markdown_field :content diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb new file mode 100644 index 00000000000..fae1b64961a --- /dev/null +++ b/app/models/storage/hashed_project.rb @@ -0,0 +1,42 @@ +module Storage + class HashedProject + attr_accessor :project + delegate :gitlab_shell, :repository_storage_path, to: :project + + ROOT_PATH_PREFIX = '@hashed'.freeze + + def initialize(project) + @project = project + end + + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash + end + + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions + def disk_path + "#{base_dir}/#{disk_hash}" if disk_hash + end + + def ensure_storage_path_exists + gitlab_shell.add_namespace(repository_storage_path, base_dir) + end + + def rename_repo + true + end + + private + + # Generates the hash for the project path and name on disk + # If you need to refer to the repository on disk, use the `#disk_path` + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id + end + end +end diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb new file mode 100644 index 00000000000..9d9e5e1d352 --- /dev/null +++ b/app/models/storage/legacy_project.rb @@ -0,0 +1,51 @@ +module Storage + class LegacyProject + attr_accessor :project + delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project + + def initialize(project) + @project = project + end + + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + namespace.full_path + end + + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions + def disk_path + project.full_path + end + + def ensure_storage_path_exists + return unless namespace + + gitlab_shell.add_namespace(repository_storage_path, base_dir) + end + + def rename_repo + new_full_path = project.build_full_path + + if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path) + # If repository moved successfully we need to send update instructions to users. + # However we cannot allow rollback since we moved repository + # So we basically we mute exceptions in next actions + begin + gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki") + return true + rescue => e + Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}" + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + return false + end + end + + false + end + end +end diff --git a/app/models/user.rb b/app/models/user.rb index 5148886eed7..d7549409b15 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -2,9 +2,11 @@ require 'carrierwave/orm/activerecord' class User < ActiveRecord::Base extend Gitlab::ConfigHelper + extend Gitlab::CurrentSettings include Gitlab::ConfigHelper include Gitlab::CurrentSettings + include Gitlab::SQL::Pattern include Avatarable include Referable include Sortable @@ -13,10 +15,12 @@ class User < ActiveRecord::Base include IgnorableColumn include FeatureGate include CreatedAtFilterable + include IgnorableColumn DEFAULT_NOTIFICATION_LEVEL = :participating - ignore_column :authorized_projects_populated + ignore_column :external_email + ignore_column :email_provider add_authentication_token_field :authentication_token add_authentication_token_field :incoming_email_token @@ -31,6 +35,7 @@ class User < ActiveRecord::Base default_value_for :project_view, :files default_value_for :notified_of_own_activity, false default_value_for :preferred_language, I18n.default_locale + default_value_for :theme_id, gitlab_config.default_theme attr_encrypted :otp_secret, key: Gitlab::Application.secrets.otp_key_base, @@ -47,11 +52,6 @@ class User < ActiveRecord::Base devise :lockable, :recoverable, :rememberable, :trackable, :validatable, :omniauthable, :confirmable, :registerable - # devise overrides #inspect, so we manually use the Referable one - def inspect - referable_inspect - end - # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour def update_tracked_fields!(request) @@ -73,7 +73,7 @@ class User < ActiveRecord::Base # # Namespace for personal projects - has_one :namespace, -> { where type: nil }, dependent: :destroy, foreign_key: :owner_id, autosave: true # rubocop:disable Cop/ActiveRecordDependent + has_one :namespace, -> { where(type: nil) }, dependent: :destroy, foreign_key: :owner_id, autosave: true # rubocop:disable Cop/ActiveRecordDependent # Profile has_many :keys, -> do @@ -88,6 +88,7 @@ class User < ActiveRecord::Base has_many :identities, dependent: :destroy, autosave: true # rubocop:disable Cop/ActiveRecordDependent has_many :u2f_registrations, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :chat_names, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :user_synced_attributes_metadata, autosave: true # Groups has_many :members, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -148,6 +149,8 @@ class User < ActiveRecord::Base uniqueness: { case_sensitive: false } validate :namespace_uniq, if: :username_changed? + validate :namespace_move_dir_allowed, if: :username_changed? + validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :unique_email, if: :email_changed? validate :owns_notification_email, if: :notification_email_changed? @@ -162,6 +165,7 @@ class User < ActiveRecord::Base after_update :update_emails_with_primary_email, if: :email_changed? before_save :ensure_authentication_token, :ensure_incoming_email_token before_save :ensure_user_rights_and_limits, if: :external_changed? + before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) } after_save :ensure_namespace_correct after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') } after_initialize :set_projects_limit @@ -256,11 +260,13 @@ class User < ActiveRecord::Base end def sort(method) - case method.to_s + order_method = method || 'id_desc' + + case order_method.to_s when 'recent_sign_in' then order_recent_sign_in when 'oldest_sign_in' then order_oldest_sign_in else - order_by(method) + order_by(order_method) end end @@ -306,7 +312,7 @@ class User < ActiveRecord::Base # Returns an ActiveRecord::Relation. def search(query) table = arel_table - pattern = "%#{query}%" + pattern = User.to_pattern(query) order = <<~SQL CASE @@ -368,7 +374,7 @@ class User < ActiveRecord::Base # Returns a user for the given SSH key. def find_by_ssh_key_id(key_id) - find_by(id: Key.unscoped.select(:user_id).where(id: key_id)) + Key.find_by(id: key_id)&.user end def find_by_full_path(path, follow_redirects: false) @@ -487,6 +493,12 @@ class User < ActiveRecord::Base end end + def namespace_move_dir_allowed + if namespace&.any_project_has_container_registry_tags? + errors.add(:username, 'cannot be changed if a personal project has container registry tags.') + end + end + def avatar_type unless avatar.image? errors.add :avatar, "only images allowed" @@ -528,7 +540,7 @@ class User < ActiveRecord::Base union = Gitlab::SQL::Union .new([groups.select(:id), authorized_projects.select(:namespace_id)]) - Group.where("namespaces.id IN (#{union.to_sql})") + Group.where("namespaces.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end # Returns a relation of groups the user has access to, including their parent @@ -598,7 +610,7 @@ class User < ActiveRecord::Base end def require_personal_access_token_creation_for_git_auth? - return false if allow_password_authentication? || ldap_user? + return false if current_application_settings.password_authentication_enabled? || ldap_user? PersonalAccessTokensFinder.new(user: self, impersonation: false, state: 'active').execute.none? end @@ -639,11 +651,6 @@ class User < ActiveRecord::Base @personal_projects_count ||= personal_projects.count end - def projects_limit_percent - return 100 if projects_limit.zero? - (personal_projects.count.to_f / projects_limit) * 100 - end - def recent_push(project_ids = nil) # Get push events not earlier than 2 hours ago events = recent_events.code_push.where("created_at > ?", Time.now - 2.hours) @@ -661,10 +668,6 @@ class User < ActiveRecord::Base end end - def projects_sorted_by_activity - authorized_projects.sorted_by_activity - end - def several_namespaces? owned_groups.any? || masters_groups.any? end @@ -718,9 +721,9 @@ class User < ActiveRecord::Base end def sanitize_attrs - %w[username skype linkedin twitter].each do |attr| - value = public_send(attr) - public_send("#{attr}=", Sanitize.clean(value)) if value.present? + %i[skype linkedin twitter].each do |attr| + value = self[attr] + self[attr] = Sanitize.clean(value) if value.present? end end @@ -779,7 +782,7 @@ class User < ActiveRecord::Base def with_defaults User.defaults.each do |k, v| - public_send("#{k}=", v) + public_send("#{k}=", v) # rubocop:disable GitlabSecurity/PublicSend end self @@ -825,7 +828,7 @@ class User < ActiveRecord::Base { name: name, username: username, - avatar_url: avatar_url + avatar_url: avatar_url(only_path: false) } end @@ -834,7 +837,12 @@ class User < ActiveRecord::Base create_namespace!(path: username, name: username) unless namespace if username_changed? - namespace.update_attributes(path: username, name: username) + unless namespace.update_attributes(path: username, name: username) + namespace.errors.each do |attribute, message| + self.errors.add(:"namespace_#{attribute}", message) + end + raise ActiveRecord::RecordInvalid.new(namespace) + end end end @@ -919,7 +927,7 @@ class User < ActiveRecord::Base def ci_authorized_runners @ci_authorized_runners ||= begin runner_ids = Ci::RunnerProject - .where("ci_runner_projects.project_id IN (#{ci_projects_union.to_sql})") + .where("ci_runner_projects.project_id IN (#{ci_projects_union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection .select(:runner_id) Ci::Runner.specific.where(id: runner_ids) end @@ -1040,6 +1048,26 @@ class User < ActiveRecord::Base ensure_rss_token! end + def verified_email?(email) + self.email == email + end + + def sync_attribute?(attribute) + return true if ldap_user? && attribute == :email + + attributes = Gitlab.config.omniauth.sync_profile_attributes + + if attributes.is_a?(Array) + attributes.include?(attribute.to_s) + else + attributes + end + end + + def read_only_attribute?(attribute) + user_synced_attributes_metadata&.read_only?(attribute) + end + protected # override, from Devise::Validatable @@ -1061,7 +1089,8 @@ class User < ActiveRecord::Base # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration def send_devise_notification(notification, *args) - devise_mailer.send(notification, self, *args).deliver_later + return true unless can?(:receive_notifications) + devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend end # This works around a bug in Devise 4.2.0 that erroneously causes a user to diff --git a/app/models/user_synced_attributes_metadata.rb b/app/models/user_synced_attributes_metadata.rb new file mode 100644 index 00000000000..9f374304164 --- /dev/null +++ b/app/models/user_synced_attributes_metadata.rb @@ -0,0 +1,25 @@ +class UserSyncedAttributesMetadata < ActiveRecord::Base + belongs_to :user + + validates :user, presence: true + + SYNCABLE_ATTRIBUTES = %i[name email location].freeze + + def read_only?(attribute) + Gitlab.config.omniauth.sync_profile_from_provider && synced?(attribute) + end + + def read_only_attributes + return [] unless Gitlab.config.omniauth.sync_profile_from_provider + + SYNCABLE_ATTRIBUTES.select { |key| synced?(key) } + end + + def synced?(attribute) + read_attribute("#{attribute}_synced") + end + + def set_attribute_synced(attribute, value) + write_attribute("#{attribute}_synced", value) + end +end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 5c7c2204374..f2315bb3dbb 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -84,7 +84,7 @@ class WikiPage # The formatted title of this page. def title if @attributes[:title] - self.class.unhyphenize(@attributes[:title]) + CGI.unescape_html(self.class.unhyphenize(@attributes[:title])) else "" end |