diff options
Diffstat (limited to 'app/models')
86 files changed, 1372 insertions, 593 deletions
diff --git a/app/models/ability.rb b/app/models/ability.rb index f3692a5a067..0b6bcbde5d9 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -1,35 +1,20 @@ +require_dependency 'declarative_policy' + class Ability class << self # Given a list of users and a project this method returns the users that can # read the given project. def users_that_can_read_project(users, project) - if project.public? - users - else - users.select do |user| - if user.admin? - true - elsif project.internal? && !user.external? - true - elsif project.owner == user - true - elsif project.team.members.include?(user) - true - else - false - end - end + DeclarativePolicy.subject_scope do + users.select { |u| allowed?(u, :read_project, project) } end end # Given a list of users and a snippet this method returns the users that can # read the given snippet. def users_that_can_read_personal_snippet(users, snippet) - case snippet.visibility_level - when Snippet::INTERNAL, Snippet::PUBLIC - users - when Snippet::PRIVATE - users.include?(snippet.author) ? [snippet.author] : [] + DeclarativePolicy.subject_scope do + users.select { |u| allowed?(u, :read_personal_snippet, snippet) } end end @@ -38,42 +23,35 @@ class Ability # issues - The issues to reduce down to those readable by the user. # user - The User for which to check the issues def issues_readable_by_user(issues, user = nil) - return issues if user && user.admin? - - issues.select { |issue| issue.visible_to_user?(user) } + DeclarativePolicy.user_scope do + issues.select { |issue| issue.visible_to_user?(user) } + end end - # TODO: make this private and use the actual abilities stuff for this def can_edit_note?(user, note) - return false if !note.editable? || !user.present? - return true if note.author == user || user.admin? - - if note.project - max_access_level = note.project.team.max_member_access(user.id) - max_access_level >= Gitlab::Access::MASTER - else - false - end + allowed?(user, :edit_note, note) end - def allowed?(user, action, subject = :global) - allowed(user, subject).include?(action) - end + def allowed?(user, action, subject = :global, opts = {}) + if subject.is_a?(Hash) + opts, subject = subject, :global + end - def allowed(user, subject = :global) - return BasePolicy::RuleSet.none if subject.nil? - return uncached_allowed(user, subject) unless RequestStore.active? + policy = policy_for(user, subject) - user_key = user ? user.id : 'anonymous' - subject_key = subject == :global ? 'global' : "#{subject.class.name}/#{subject.id}" - key = "/ability/#{user_key}/#{subject_key}" - RequestStore[key] ||= uncached_allowed(user, subject).freeze + case opts[:scope] + when :user + DeclarativePolicy.user_scope { policy.can?(action) } + when :subject + DeclarativePolicy.subject_scope { policy.can?(action) } + else + policy.can?(action) + end end - private - - def uncached_allowed(user, subject) - BasePolicy.class_for(subject).abilities(user, subject) + def policy_for(user, subject = :global) + cache = RequestStore.active? ? RequestStore : {} + DeclarativePolicy.policy_for(user, subject, cache: cache) end end end diff --git a/app/models/appearance.rb b/app/models/appearance.rb index c79326e8427..f9c48482be7 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -10,5 +10,5 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader - has_many :uploads, as: :model, dependent: :destroy + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 668caef0d2c..bd7c4cd45ea 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -13,13 +13,13 @@ class ApplicationSetting < ActiveRecord::Base [\r\n] # any number of newline characters }x - serialize :restricted_visibility_levels # rubocop:disable Cop/ActiverecordSerialize - serialize :import_sources # rubocop:disable Cop/ActiverecordSerialize - serialize :disabled_oauth_sign_in_sources, Array # rubocop:disable Cop/ActiverecordSerialize - serialize :domain_whitelist, Array # rubocop:disable Cop/ActiverecordSerialize - serialize :domain_blacklist, Array # rubocop:disable Cop/ActiverecordSerialize - serialize :repository_storages # rubocop:disable Cop/ActiverecordSerialize - serialize :sidekiq_throttling_queues, Array # rubocop:disable Cop/ActiverecordSerialize + serialize :restricted_visibility_levels # rubocop:disable Cop/ActiveRecordSerialize + serialize :import_sources # rubocop:disable Cop/ActiveRecordSerialize + serialize :disabled_oauth_sign_in_sources, Array # rubocop:disable Cop/ActiveRecordSerialize + serialize :domain_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize + serialize :domain_blacklist, Array # rubocop:disable Cop/ActiveRecordSerialize + serialize :repository_storages # rubocop:disable Cop/ActiveRecordSerialize + serialize :sidekiq_throttling_queues, Array # rubocop:disable Cop/ActiveRecordSerialize cache_markdown_field :sign_in_text cache_markdown_field :help_page_text @@ -184,6 +184,9 @@ class ApplicationSetting < ActiveRecord::Base Rails.cache.fetch(CACHE_KEY) do ApplicationSetting.last end + rescue + # Fall back to an uncached value if there are any problems (e.g. redis down) + ApplicationSetting.last end def self.expire @@ -234,6 +237,8 @@ class ApplicationSetting < ActiveRecord::Base koding_url: nil, max_artifacts_size: Settings.artifacts['max_size'], max_attachment_size: Settings.gitlab['max_attachment_size'], + password_authentication_enabled: Settings.gitlab['password_authentication_enabled'], + performance_bar_allowed_group_id: nil, plantuml_enabled: false, plantuml_url: nil, recaptcha_enabled: false, @@ -247,7 +252,6 @@ class ApplicationSetting < ActiveRecord::Base shared_runners_text: nil, sidekiq_throttling_enabled: false, sign_in_text: nil, - signin_enabled: Settings.gitlab['signin_enabled'], signup_enabled: Settings.gitlab['signup_enabled'], terminal_max_session_time: 0, two_factor_grace_period: 48, @@ -311,7 +315,9 @@ class ApplicationSetting < ActiveRecord::Base Array(read_attribute(:repository_storages)) end + # DEPRECATED # repository_storage is still required in the API. Remove in 9.0 + # Still used in API v3 def repository_storage repository_storages.first end @@ -336,6 +342,48 @@ class ApplicationSetting < ActiveRecord::Base super(levels.map { |level| Gitlab::VisibilityLevel.level_value(level) }) end + def performance_bar_allowed_group_id=(group_full_path) + group_full_path = nil if group_full_path.blank? + + if group_full_path.nil? + if group_full_path != performance_bar_allowed_group_id + super(group_full_path) + Gitlab::PerformanceBar.expire_allowed_user_ids_cache + end + return + end + + group = Group.find_by_full_path(group_full_path) + + if group + if group.id != performance_bar_allowed_group_id + super(group.id) + Gitlab::PerformanceBar.expire_allowed_user_ids_cache + end + else + super(nil) + Gitlab::PerformanceBar.expire_allowed_user_ids_cache + end + end + + def performance_bar_allowed_group + Group.find_by_id(performance_bar_allowed_group_id) + end + + # Return true if the Performance Bar is enabled for a given group + def performance_bar_enabled + performance_bar_allowed_group_id.present? + end + + # - If `enable` is true, we early return since the actual attribute that holds + # the enabling/disabling is `performance_bar_allowed_group_id` + # - If `enable` is false, we set `performance_bar_allowed_group_id` to `nil` + def performance_bar_enabled=(enable) + return if enable + + self.performance_bar_allowed_group_id = nil + end + # Choose one of the available repository storage options. Currently all have # equal weighting. def pick_repository_storage diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 46d412fbd72..112a8778b4e 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -1,5 +1,5 @@ class AuditEvent < ActiveRecord::Base - serialize :details, Hash # rubocop:disable Cop/ActiverecordSerialize + serialize :details, Hash # rubocop:disable Cop/ActiveRecordSerialize belongs_to :user, foreign_key: :author_id diff --git a/app/models/blob_viewer/readme.rb b/app/models/blob_viewer/readme.rb index 75c373a03bb..4604a9934a0 100644 --- a/app/models/blob_viewer/readme.rb +++ b/app/models/blob_viewer/readme.rb @@ -10,5 +10,11 @@ module BlobViewer def visible_to?(current_user) can?(current_user, :read_wiki, project) end + + def render_error + return if project.has_external_wiki? || (project.wiki_enabled? && project.wiki.has_home_page?) + + :no_wiki + end end end diff --git a/app/models/board.rb b/app/models/board.rb index 18081a32157..97d0f550925 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -1,7 +1,7 @@ class Board < ActiveRecord::Base belongs_to :project - has_many :lists, -> { order(:list_type, :position) }, dependent: :delete_all + has_many :lists, -> { order(:list_type, :position) }, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent validates :project, presence: true diff --git a/app/models/chat_team.rb b/app/models/chat_team.rb index c52b6f15913..25ecf2d5937 100644 --- a/app/models/chat_team.rb +++ b/app/models/chat_team.rb @@ -3,4 +3,13 @@ class ChatTeam < ActiveRecord::Base validates :namespace, uniqueness: true belongs_to :namespace + + def remove_mattermost_team(current_user) + Mattermost::Team.new(current_user).destroy(team_id: team_id) + rescue Mattermost::ClientError => e + # Either the group is not found, or the user doesn't have the proper + # access on the mattermost instance. In the first case, we're done either way + # in the latter case, we can't recover by retrying, so we just log what happened + Rails.logger.error("Mattermost team deletion failed: #{e}") + end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index a300536532b..8be2dee6479 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -19,8 +19,8 @@ module Ci ) end - serialize :options # rubocop:disable Cop/ActiverecordSerialize - serialize :yaml_variables, Gitlab::Serializer::Ci::Variables # rubocop:disable Cop/ActiverecordSerialize + serialize :options # rubocop:disable Cop/ActiveRecordSerialize + serialize :yaml_variables, Gitlab::Serializer::Ci::Variables # rubocop:disable Cop/ActiveRecordSerialize delegate :name, to: :project, prefix: true @@ -96,6 +96,14 @@ module Ci BuildSuccessWorker.perform_async(id) end end + + before_transition any => [:failed] do |build| + next if build.retries_max.zero? + + if build.retries_count < build.retries_max + Ci::Build.retry(build, build.user) + end + end end def detailed_status(current_user) @@ -130,6 +138,14 @@ module Ci success? || failed? || canceled? end + def retries_count + pipeline.builds.retried.where(name: self.name).count + end + + def retries_max + self.options.fetch(:retry, 0).to_i + end + def latest? !retried? end @@ -176,13 +192,22 @@ module Ci # * Lowercased # * Anything not matching [a-z0-9-] is replaced with a - # * Maximum length is 63 bytes + # * First/Last Character is not a hyphen def ref_slug - slugified = ref.to_s.downcase - slugified.gsub(/[^a-z0-9]/, '-')[0..62] + ref.to_s + .downcase + .gsub(/[^a-z0-9]/, '-')[0..62] + .gsub(/(\A-+|-+\z)/, '') end # Variables whose value does not depend on environment def simple_variables + variables(environment: nil) + end + + # All variables, including those dependent on environment, which could + # contain unexpanded variables. + def variables(environment: persisted_environment) variables = predefined_variables variables += project.predefined_variables variables += pipeline.predefined_variables @@ -191,15 +216,14 @@ module Ci variables += project.deployment_variables if has_environment? variables += yaml_variables variables += user_variables - variables += project.secret_variables_for(ref).map(&:to_runner_variable) + variables += project.group.secret_variables_for(ref, project).map(&:to_runner_variable) if project.group + variables += secret_variables(environment: environment) variables += trigger_request.user_variables if trigger_request - variables - end + variables += pipeline.variables.map(&:to_runner_variable) + variables += pipeline.pipeline_schedule.job_variables if pipeline.pipeline_schedule + variables += persisted_environment_variables if environment - # All variables, including those dependent on environment, which could - # contain unexpanded variables. - def variables - simple_variables.concat(persisted_environment_variables) + variables end def merge_request @@ -213,7 +237,7 @@ module Ci .reorder(iid: :desc) merge_requests.find do |merge_request| - merge_request.commits_sha.include?(pipeline.sha) + merge_request.commit_shas.include?(pipeline.sha) end end end @@ -367,6 +391,11 @@ module Ci ] end + def secret_variables(environment: persisted_environment) + project.secret_variables_for(ref: ref, environment: environment) + .map(&:to_runner_variable) + end + def steps [Gitlab::Ci::Build::Step.from_commands(self), Gitlab::Ci::Build::Step.from_after_script(self)].compact diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb new file mode 100644 index 00000000000..f64bc245a67 --- /dev/null +++ b/app/models/ci/group_variable.rb @@ -0,0 +1,13 @@ +module Ci + class GroupVariable < ActiveRecord::Base + extend Ci::Model + include HasVariable + include Presentable + + belongs_to :group + + validates :key, uniqueness: { scope: :group_id } + + scope :unprotected, -> { where(protected: false) } + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 1b3e5a25ac2..d2abcf30034 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,14 +14,15 @@ module Ci has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id has_many :builds, foreign_key: :commit_id - has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id + has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent + has_many :variables, class_name: 'Ci::PipelineVariable' # Merge requests for which the current pipeline is running against # the merge request's latest commit. has_many :merge_requests, foreign_key: "head_pipeline_id" has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build' - has_many :retryable_builds, -> { latest.failed_or_canceled }, foreign_key: :commit_id, class_name: 'Ci::Build' + has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build' has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus' has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build' has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build' @@ -140,6 +141,7 @@ module Ci where(id: max_id) end end + scope :internal, -> { where(source: internal_sources) } def self.latest_status(ref = nil) latest(ref).status @@ -163,6 +165,10 @@ module Ci where.not(duration: nil).sum(:duration) end + def self.internal_sources + sources.reject { |source| source == "external" }.values + end + def stages_count statuses.select(:stage).distinct.count end @@ -321,10 +327,24 @@ module Ci end end + def ci_yaml_file_path + if project.ci_config_path.blank? + '.gitlab-ci.yml' + else + project.ci_config_path + end + end + def ci_yaml_file return @ci_yaml_file if defined?(@ci_yaml_file) - @ci_yaml_file = project.repository.gitlab_ci_yml_for(sha) rescue nil + @ci_yaml_file = begin + project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path) + rescue Rugged::ReferenceError, GRPC::NotFound, GRPC::Internal + self.yaml_errors = + "Failed to load CI/CD config file at #{ci_yaml_file_path}" + nil + end end def has_yaml_errors? @@ -372,7 +392,8 @@ module Ci def predefined_variables [ - { key: 'CI_PIPELINE_ID', value: id.to_s, public: true } + { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }, + { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true } ] end diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 45d8cd34359..085eeeae157 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -9,17 +9,21 @@ module Ci belongs_to :owner, class_name: 'User' has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline' has_many :pipelines + has_many :variables, class_name: 'Ci::PipelineScheduleVariable' validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? } validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } validates :description, presence: true + validates :variables, variable_duplicates: true before_save :set_next_run_at scope :active, -> { where(active: true) } scope :inactive, -> { where(active: false) } + accepts_nested_attributes_for :variables, allow_destroy: true + def owned_by?(current_user) owner == current_user end @@ -36,10 +40,6 @@ module Ci update_attribute(:active, false) end - def runnable_by_owner? - Ability.allowed?(owner, :create_pipeline, project) - end - def set_next_run_at self.next_run_at = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now) end @@ -56,5 +56,9 @@ module Ci Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone) .next_time_from(next_run_at) end + + def job_variables + variables&.map(&:to_runner_variable) || [] + end end end diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb new file mode 100644 index 00000000000..1ff177616e8 --- /dev/null +++ b/app/models/ci/pipeline_schedule_variable.rb @@ -0,0 +1,8 @@ +module Ci + class PipelineScheduleVariable < ActiveRecord::Base + extend Ci::Model + include HasVariable + + belongs_to :pipeline_schedule + end +end diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb new file mode 100644 index 00000000000..00b419c3efa --- /dev/null +++ b/app/models/ci/pipeline_variable.rb @@ -0,0 +1,10 @@ +module Ci + class PipelineVariable < ActiveRecord::Base + extend Ci::Model + include HasVariable + + belongs_to :pipeline + + validates :key, uniqueness: { scope: :pipeline_id } + end +end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index d12f96f3d0b..c6d23898560 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -3,12 +3,12 @@ module Ci extend Ci::Model RUNNER_QUEUE_EXPIRY_TIME = 60.minutes - LAST_CONTACT_TIME = 1.hour.ago + ONLINE_CONTACT_TIMEOUT = 1.hour AVAILABLE_SCOPES = %w[specific shared active paused online].freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked].freeze has_many :builds - has_many :runner_projects, dependent: :destroy + has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' @@ -19,7 +19,7 @@ module Ci scope :shared, ->() { where(is_shared: true) } scope :active, ->() { where(active: true) } scope :paused, ->() { where(active: false) } - scope :online, ->() { where('contacted_at > ?', LAST_CONTACT_TIME) } + scope :online, ->() { where('contacted_at > ?', contact_time_deadline) } scope :ordered, ->() { order(id: :desc) } scope :owned_or_shared, ->(project_id) do @@ -59,6 +59,10 @@ module Ci where(t[:token].matches(pattern).or(t[:description].matches(pattern))) end + def self.contact_time_deadline + ONLINE_CONTACT_TIMEOUT.ago + end + def set_default_values self.token = SecureRandom.hex(15) if self.token.blank? end @@ -80,7 +84,7 @@ module Ci end def online? - contacted_at && contacted_at > LAST_CONTACT_TIME + contacted_at && contacted_at > self.class.contact_time_deadline end def status @@ -145,7 +149,7 @@ module Ci private def cleanup_runner_queue - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| redis.del(runner_queue_key) end end diff --git a/app/models/ci/trigger_request.rb b/app/models/ci/trigger_request.rb index 564334ad1ad..c58ce5c3717 100644 --- a/app/models/ci/trigger_request.rb +++ b/app/models/ci/trigger_request.rb @@ -6,7 +6,7 @@ module Ci belongs_to :pipeline, foreign_key: :commit_id has_many :builds - serialize :variables # rubocop:disable Cop/ActiverecordSerialize + serialize :variables # rubocop:disable Cop/ActiveRecordSerialize def user_variables return [] unless variables diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index f235260208f..cf0fe04ddaf 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -1,27 +1,13 @@ module Ci class Variable < ActiveRecord::Base extend Ci::Model + include HasVariable + include Presentable belongs_to :project - validates :key, - presence: true, - uniqueness: { scope: :project_id }, - length: { maximum: 255 }, - format: { with: /\A[a-zA-Z0-9_]+\z/, - message: "can contain only letters, digits and '_'." } + validates :key, uniqueness: { scope: [:project_id, :environment_scope] } - scope :order_key_asc, -> { reorder(key: :asc) } scope :unprotected, -> { where(protected: false) } - - attr_encrypted :value, - mode: :per_attribute_iv_and_salt, - insecure_mode: true, - key: Gitlab::Application.secrets.db_key_base, - algorithm: 'aes-256-cbc' - - def to_runner_variable - { key: key, value: value, public: false } - end end end diff --git a/app/models/commit.rb b/app/models/commit.rb index 20206d57c4c..7940733f557 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -1,5 +1,6 @@ class Commit extend ActiveModel::Naming + extend Gitlab::Cache::RequestCache include ActiveModel::Conversion include Noteable @@ -138,7 +139,7 @@ class Commit safe_message.split("\n", 2)[1].try(:chomp) end - + def description? description.present? end @@ -169,19 +170,9 @@ class Commit end def author - if RequestStore.active? - key = "commit_author:#{author_email.downcase}" - # nil is a valid value since no author may exist in the system - if RequestStore.store.key?(key) - @author = RequestStore.store[key] - else - @author = find_author_by_any_email - RequestStore.store[key] = @author - end - else - @author ||= find_author_by_any_email - end + User.find_by_any_email(author_email.downcase) end + request_cache(:author) { author_email.downcase } def committer @committer ||= User.find_by_any_email(committer_email.downcase) @@ -243,6 +234,14 @@ class Commit @statuses[ref] = pipelines.latest_status(ref) end + def signature + return @signature if defined?(@signature) + + @signature = gpg_commit.signature + end + + delegate :has_signature?, to: :gpg_commit + def revert_branch_name "revert-#{short_id}" end @@ -322,7 +321,7 @@ class Commit def raw_diffs(*args) if Gitlab::GitalyClient.feature_enabled?(:commit_raw_diffs) - Gitlab::GitalyClient::Commit.new(project.repository).diff_from_parent(self, *args) + Gitlab::GitalyClient::CommitService.new(project.repository).diff_from_parent(self, *args) else raw.diffs(*args) end @@ -331,7 +330,7 @@ class Commit def raw_deltas @deltas ||= Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled| if is_enabled - Gitlab::GitalyClient::Commit.new(project.repository).commit_deltas(self) + Gitlab::GitalyClient::CommitService.new(project.repository).commit_deltas(self) else raw.deltas end @@ -368,10 +367,6 @@ class Commit end end - def find_author_by_any_email - User.find_by_any_email(author_email.downcase) - end - def repo_changes changes = { added: [], modified: [], removed: [] } @@ -395,4 +390,8 @@ class Commit def merged_merge_request_no_cache(user) MergeRequestsFinder.new(user, project_id: project.id).find_by(merge_commit_sha: id) if merge_commit? end + + def gpg_commit + @gpg_commit ||= Gitlab::Gpg::Commit.new(self) + end end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index a7fd0a15f0f..f4f9b037957 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -2,7 +2,7 @@ module Awardable extend ActiveSupport::Concern included do - has_many :award_emoji, -> { includes(:user).order(:id) }, as: :awardable, dependent: :destroy + has_many :award_emoji, -> { includes(:user).order(:id) }, as: :awardable, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent if self < Participable # By default we always load award_emoji user association diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index eb32bf3d32a..48547a938fc 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -11,7 +11,7 @@ module CacheMarkdownField extend ActiveSupport::Concern # Increment this number every time the renderer changes its output - CACHE_VERSION = 1 + CACHE_VERSION = 2 # changes to these attributes cause the cache to be invalidates INVALIDATED_BY = %w[author project].freeze @@ -78,7 +78,7 @@ module CacheMarkdownField def cached_html_up_to_date?(markdown_field) html_field = cached_markdown_fields.html_field(markdown_field) - cached = !cached_html_for(markdown_field).nil? && !__send__(markdown_field).nil? + cached = cached_html_for(markdown_field).present? && __send__(markdown_field).present? return false unless cached markdown_changed = attribute_changed?(markdown_field) || false diff --git a/app/models/concerns/created_at_filterable.rb b/app/models/concerns/created_at_filterable.rb new file mode 100644 index 00000000000..e8a3e41203d --- /dev/null +++ b/app/models/concerns/created_at_filterable.rb @@ -0,0 +1,12 @@ +module CreatedAtFilterable + extend ActiveSupport::Concern + + included do + scope :created_before, ->(date) { where(scoped_table[:created_at].lteq(date)) } + scope :created_after, ->(date) { where(scoped_table[:created_at].gteq(date)) } + + def self.scoped_table + arel_table.alias(table_name) + end + end +end diff --git a/app/models/concerns/each_batch.rb b/app/models/concerns/each_batch.rb new file mode 100644 index 00000000000..6ddbb8da1a9 --- /dev/null +++ b/app/models/concerns/each_batch.rb @@ -0,0 +1,81 @@ +module EachBatch + extend ActiveSupport::Concern + + module ClassMethods + # Iterates over the rows in a relation in batches, similar to Rails' + # `in_batches` but in a more efficient way. + # + # Unlike `in_batches` provided by Rails this method does not support a + # custom start/end range, nor does it provide support for the `load:` + # keyword argument. + # + # This method will yield an ActiveRecord::Relation to the supplied block, or + # return an Enumerator if no block is given. + # + # Example: + # + # User.each_batch do |relation| + # relation.update_all(updated_at: Time.now) + # end + # + # The supplied block is also passed an optional batch index: + # + # User.each_batch do |relation, index| + # puts index # => 1, 2, 3, ... + # end + # + # You can also specify an alternative column to use for ordering the rows: + # + # User.each_batch(column: :created_at) do |relation| + # ... + # end + # + # This will produce SQL queries along the lines of: + # + # User Load (0.7ms) SELECT "users"."id" FROM "users" WHERE ("users"."id" >= 41654) ORDER BY "users"."id" ASC LIMIT 1 OFFSET 1000 + # (0.7ms) SELECT COUNT(*) FROM "users" WHERE ("users"."id" >= 41654) AND ("users"."id" < 42687) + # + # of - The number of rows to retrieve per batch. + # column - The column to use for ordering the batches. + def each_batch(of: 1000, column: primary_key) + unless column + raise ArgumentError, + 'the column: argument must be set to a column name to use for ordering rows' + end + + start = except(:select) + .select(column) + .reorder(column => :asc) + .take + + return unless start + + start_id = start[column] + arel_table = self.arel_table + + 1.step do |index| + stop = except(:select) + .select(column) + .where(arel_table[column].gteq(start_id)) + .reorder(column => :asc) + .offset(of) + .limit(1) + .take + + relation = where(arel_table[column].gteq(start_id)) + + if stop + stop_id = stop[column] + start_id = stop_id + relation = relation.where(arel_table[column].lt(stop_id)) + end + + # Any ORDER BYs are useless for this relation and can lead to less + # efficient UPDATE queries, hence we get rid of it. + yield relation.except(:order), index + + break unless stop + end + end + end +end diff --git a/app/models/concerns/editable.rb b/app/models/concerns/editable.rb index c62c7e1e936..28623d257a6 100644 --- a/app/models/concerns/editable.rb +++ b/app/models/concerns/editable.rb @@ -4,4 +4,8 @@ module Editable def is_edited? last_edited_at.present? && last_edited_at != created_at end + + def last_edited_by + super || User.ghost + end end diff --git a/app/models/concerns/feature_gate.rb b/app/models/concerns/feature_gate.rb new file mode 100644 index 00000000000..5db64fe82c4 --- /dev/null +++ b/app/models/concerns/feature_gate.rb @@ -0,0 +1,7 @@ +module FeatureGate + def flipper_id + return nil if new_record? + + "#{self.class.name}:#{id}" + end +end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 3c9c6584e02..32af5566135 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -11,18 +11,21 @@ module HasStatus class_methods do def status_sql - scope = respond_to?(:exclude_ignored) ? exclude_ignored : all - - builds = scope.select('count(*)').to_sql - created = scope.created.select('count(*)').to_sql - success = scope.success.select('count(*)').to_sql - manual = scope.manual.select('count(*)').to_sql - pending = scope.pending.select('count(*)').to_sql - running = scope.running.select('count(*)').to_sql - skipped = scope.skipped.select('count(*)').to_sql - canceled = scope.canceled.select('count(*)').to_sql + scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all + scope_warnings = respond_to?(:failed_but_allowed) ? failed_but_allowed : none + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql.presence || 'false' "(CASE + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' WHEN (#{builds})=(#{skipped}) THEN 'skipped' WHEN (#{builds})=(#{success}) THEN 'success' WHEN (#{builds})=(#{created}) THEN 'created' diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb new file mode 100644 index 00000000000..9585b5583dc --- /dev/null +++ b/app/models/concerns/has_variable.rb @@ -0,0 +1,23 @@ +module HasVariable + extend ActiveSupport::Concern + + included do + validates :key, + presence: true, + length: { maximum: 255 }, + format: { with: /\A[a-zA-Z0-9_]+\z/, + message: "can contain only letters, digits and '_'." } + + scope :order_key_asc, -> { reorder(key: :asc) } + + attr_encrypted :value, + mode: :per_attribute_iv_and_salt, + insecure_mode: true, + key: Gitlab::Application.secrets.db_key_base, + algorithm: 'aes-256-cbc' + + def to_runner_variable + { key: key, value: value, public: false } + end + end +end diff --git a/app/models/concerns/internal_id.rb b/app/models/concerns/internal_id.rb index 5382dde6765..67a0adfcd56 100644 --- a/app/models/concerns/internal_id.rb +++ b/app/models/concerns/internal_id.rb @@ -8,7 +8,8 @@ module InternalId def set_iid if iid.blank? - records = project.send(self.class.name.tableize) + parent = project || group + records = parent.send(self.class.name.tableize) records = records.with_deleted if self.paranoid? max_iid = records.maximum(:iid) diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index d178ee4422b..935ffe343ff 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -30,7 +30,8 @@ module Issuable belongs_to :updated_by, class_name: "User" belongs_to :last_edited_by, class_name: 'User' belongs_to :milestone - has_many :notes, as: :noteable, inverse_of: :noteable, dependent: :destroy do + + has_many :notes, as: :noteable, inverse_of: :noteable, dependent: :destroy do # rubocop:disable Cop/ActiveRecordDependent def authors_loaded? # We check first if we're loaded to not load unnecessarily. loaded? && to_a.all? { |note| note.association(:author).loaded? } @@ -42,9 +43,9 @@ module Issuable end end - has_many :label_links, as: :target, dependent: :destroy + has_many :label_links, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :labels, through: :label_links - has_many :todos, as: :target, dependent: :destroy + has_many :todos, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :metrics @@ -70,9 +71,8 @@ module Issuable scope :of_projects, ->(ids) { where(project_id: ids) } scope :of_milestones, ->(ids) { where(milestone_id: ids) } scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) } - scope :opened, -> { with_state(:opened, :reopened) } + scope :opened, -> { with_state(:opened) } scope :only_opened, -> { with_state(:opened) } - scope :only_reopened, -> { with_state(:reopened) } scope :closed, -> { with_state(:closed) } scope :left_joins_milestones, -> { joins("LEFT OUTER JOIN milestones ON #{table_name}.milestone_id = milestones.id") } @@ -102,6 +102,14 @@ module Issuable def locking_enabled? title_changed? || description_changed? end + + def allows_multiple_assignees? + false + end + + def has_multiple_assignees? + assignees.count > 1 + end end module ClassMethods @@ -225,7 +233,7 @@ module Issuable end def open? - opened? || reopened? + opened? end def user_notes_count diff --git a/app/models/concerns/mentionable/reference_regexes.rb b/app/models/concerns/mentionable/reference_regexes.rb index 1848230ec7e..2d86a70c395 100644 --- a/app/models/concerns/mentionable/reference_regexes.rb +++ b/app/models/concerns/mentionable/reference_regexes.rb @@ -14,7 +14,7 @@ module Mentionable end EXTERNAL_PATTERN = begin - issue_pattern = ExternalIssue.reference_pattern + issue_pattern = IssueTrackerService.reference_pattern link_patterns = URI.regexp(%w(http https)) reference_pattern(link_patterns, issue_pattern) end diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index 01599ce49c6..f0998465822 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -70,6 +70,22 @@ module Milestoneish due_date && due_date.past? end + def is_group_milestone? + false + end + + def is_project_milestone? + false + end + + def is_legacy_group_milestone? + false + end + + def is_dashboard_milestone? + false + end + private def count_issues_by_state(user) diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index 47e71c58557..ef95d6b0f98 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -17,7 +17,13 @@ module ProtectedRef class_methods do def protected_ref_access_levels(*types) types.each do |type| - has_many :"#{type}_access_levels", dependent: :destroy + # We need to set `inverse_of` to make sure the `belongs_to`-object is set + # when creating children using `accepts_nested_attributes_for`. + # + # If we don't `protected_branch` or `protected_tag` would be empty and + # `project` cannot be delegated to it, which in turn would cause validations + # to fail. + has_many :"#{type}_access_levels", dependent: :destroy, inverse_of: self.model_name.singular # rubocop:disable Cop/ActiveRecordDependent validates :"#{type}_access_levels", length: { is: 1, message: "are restricted to a single instance per #{self.model_name.human}." } @@ -25,8 +31,8 @@ module ProtectedRef end end - def protected_ref_accessible_to?(ref, user, action:) - access_levels_for_ref(ref, action: action).any? do |access_level| + def protected_ref_accessible_to?(ref, user, action:, protected_refs: nil) + access_levels_for_ref(ref, action: action, protected_refs: protected_refs).any? do |access_level| access_level.check_access(user) end end @@ -37,8 +43,9 @@ module ProtectedRef end end - def access_levels_for_ref(ref, action:) - self.matching(ref).map(&:"#{action}_access_levels").flatten + def access_levels_for_ref(ref, action:, protected_refs: nil) + self.matching(ref, protected_refs: protected_refs) + .map(&:"#{action}_access_levels").flatten end def matching(ref_name, protected_refs: nil) diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index ec7796a9dbb..f5048d17d80 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -4,8 +4,8 @@ module Routable extend ActiveSupport::Concern included do - has_one :route, as: :source, autosave: true, dependent: :destroy - has_many :redirect_routes, as: :source, autosave: true, dependent: :destroy + has_one :route, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :redirect_routes, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent validates_associated :route validates :route, presence: true @@ -103,8 +103,12 @@ module Routable def full_path return uncached_full_path unless RequestStore.active? - key = "routable/full_path/#{self.class.name}/#{self.id}" - RequestStore[key] ||= uncached_full_path + RequestStore[full_path_key] ||= uncached_full_path + end + + def expires_full_path_cache + RequestStore.delete(full_path_key) if RequestStore.active? + @full_path = nil end def build_full_path @@ -135,6 +139,10 @@ module Routable path_changed? || parent_changed? end + def full_path_key + @full_path_key ||= "routable/full_path/#{self.class.name}/#{self.id}" + end + def build_full_name if parent && name parent.human_name + ' / ' + name diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb new file mode 100644 index 00000000000..67ecf470f7e --- /dev/null +++ b/app/models/concerns/sha_attribute.rb @@ -0,0 +1,20 @@ +module ShaAttribute + extend ActiveSupport::Concern + + module ClassMethods + def sha_attribute(name) + return unless table_exists? + + column = columns.find { |c| c.name == name.to_s } + + # In case the table doesn't exist we won't be able to find the column, + # thus we will only check the type if the column is present. + if column && column.type != :binary + raise ArgumentError, + "sha_attribute #{name.inspect} is invalid since the column type is not :binary" + end + + attribute(name, Gitlab::Database::ShaAttribute.new) + end + end +end diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index 647a6cad3d7..bd75f25a210 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -8,7 +8,7 @@ module Spammable end included do - has_one :user_agent_detail, as: :subject, dependent: :destroy + has_one :user_agent_detail, as: :subject, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent attr_accessor :spam attr_accessor :spam_log diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index f60a0f8f438..274b38a7708 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -9,7 +9,7 @@ module Subscribable extend ActiveSupport::Concern included do - has_many :subscriptions, dependent: :destroy, as: :subscribable + has_many :subscriptions, dependent: :destroy, as: :subscribable # rubocop:disable Cop/ActiveRecordDependent end def subscribed?(user, project = nil) diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 9cf83440784..b517ddaebd7 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -18,7 +18,7 @@ module TimeTrackable validates :time_estimate, numericality: { message: 'has an invalid format' }, allow_nil: false validate :check_negative_time_spent - has_many :timelogs, dependent: :destroy + has_many :timelogs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent end def spend_time(options) diff --git a/app/models/dashboard_milestone.rb b/app/models/dashboard_milestone.rb index 646c1e5ce1a..fac7c5e5c85 100644 --- a/app/models/dashboard_milestone.rb +++ b/app/models/dashboard_milestone.rb @@ -2,4 +2,8 @@ class DashboardMilestone < GlobalMilestone def issues_finder_params { authorized_only: true } end + + def is_dashboard_milestone? + true + end end diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 053f2a11aa0..51768dd96bc 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -1,5 +1,5 @@ class DeployKey < Key - has_many :deploy_keys_projects, dependent: :destroy + has_many :deploy_keys_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :deploy_keys_projects scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where('deploy_keys_projects.project_id in (?)', projects) } diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 32cfa935aa7..056c49e7162 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -114,6 +114,17 @@ class Deployment < ActiveRecord::Base project.monitoring_service.deployment_metrics(self) end + def has_additional_metrics? + project.prometheus_service.present? + end + + def additional_metrics + return {} unless project.prometheus_service.present? + + metrics = project.prometheus_service.additional_deployment_metrics(self) + metrics&.merge(deployment_time: created_at.to_i) || {} + end + private def ref_path diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index 20ef1378500..e9a60e6ce09 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -6,9 +6,9 @@ class DiffNote < Note NOTEABLE_TYPES = %w(MergeRequest Commit).freeze - serialize :original_position, Gitlab::Diff::Position # rubocop:disable Cop/ActiverecordSerialize - serialize :position, Gitlab::Diff::Position # rubocop:disable Cop/ActiverecordSerialize - serialize :change_position, Gitlab::Diff::Position # rubocop:disable Cop/ActiverecordSerialize + serialize :original_position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize + serialize :position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize + serialize :change_position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize validates :original_position, presence: true validates :position, presence: true diff --git a/app/models/environment.rb b/app/models/environment.rb index 781cba76e3c..e9ebf0637f3 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -6,7 +6,7 @@ class Environment < ActiveRecord::Base belongs_to :project, required: true, validate: true - has_many :deployments, dependent: :destroy + has_many :deployments, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :last_deployment, -> { order('deployments.id DESC') }, class_name: 'Deployment' before_validation :nullify_external_url @@ -45,6 +45,7 @@ class Environment < ActiveRecord::Base .to_sql order(Gitlab::Database.nulls_first_order("(#{max_deployment_id_sql})", 'ASC')) end + scope :in_review_folder, -> { where(environment_type: "review") } state_machine :state, initial: :available do event :start do @@ -157,6 +158,16 @@ class Environment < ActiveRecord::Base project.monitoring_service.environment_metrics(self) if has_metrics? end + def has_additional_metrics? + project.prometheus_service.present? && available? && last_deployment.present? + end + + def additional_metrics + if has_additional_metrics? + project.prometheus_service.additional_environment_metrics(self) + end + end + # An environment name is not necessarily suitable for use in URLs, DNS # or other third-party contexts, so provide a slugified version. A slug has # the following properties: @@ -207,8 +218,7 @@ class Environment < ActiveRecord::Base end def etag_cache_key - Gitlab::Routing.url_helpers.namespace_project_environments_path( - project.namespace, + Gitlab::Routing.url_helpers.project_environments_path( project, format: :json) end diff --git a/app/models/event.rb b/app/models/event.rb index 29bc141c5cd..8d93a228494 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -50,7 +50,7 @@ class Event < ActiveRecord::Base belongs_to :target, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations # For Hash only - serialize :data # rubocop:disable Cop/ActiverecordSerialize + serialize :data # rubocop:disable Cop/ActiveRecordSerialize # Callbacks after_create :reset_project_activity diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb index e63f89a9f85..0bf18e529f0 100644 --- a/app/models/external_issue.rb +++ b/app/models/external_issue.rb @@ -38,11 +38,6 @@ class ExternalIssue @project.id end - # Pattern used to extract `JIRA-123` issue references from text - def self.reference_pattern - @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} - end - def to_reference(_from_project = nil, full: nil) id end diff --git a/app/models/forked_project_link.rb b/app/models/forked_project_link.rb index 36cf7ad6a28..8d35864eff6 100644 --- a/app/models/forked_project_link.rb +++ b/app/models/forked_project_link.rb @@ -1,4 +1,4 @@ class ForkedProjectLink < ActiveRecord::Base - belongs_to :forked_to_project, class_name: 'Project' - belongs_to :forked_from_project, class_name: 'Project' + belongs_to :forked_to_project, -> { where.not(pending_delete: true) }, class_name: 'Project' + belongs_to :forked_from_project, -> { where.not(pending_delete: true) }, class_name: 'Project' end diff --git a/app/models/global_label.rb b/app/models/global_label.rb index 698a7bbd327..2a1b7564962 100644 --- a/app/models/global_label.rb +++ b/app/models/global_label.rb @@ -2,7 +2,7 @@ class GlobalLabel attr_accessor :title, :labels alias_attribute :name, :title - delegate :color, :description, to: :@first_label + delegate :color, :text_color, :description, to: :@first_label def for_display @first_label diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb index 538615130a7..c0864769314 100644 --- a/app/models/global_milestone.rb +++ b/app/models/global_milestone.rb @@ -2,6 +2,7 @@ class GlobalMilestone include Milestoneish EPOCH = DateTime.parse('1970-01-01') + STATE_COUNT_HASH = { opened: 0, closed: 0, all: 0 }.freeze attr_accessor :title, :milestones alias_attribute :name, :title @@ -11,7 +12,10 @@ class GlobalMilestone end def self.build_collection(projects, params) - child_milestones = MilestonesFinder.new.execute(projects, params) + params = + { project_ids: projects.map(&:id), state: params[:state] } + + child_milestones = MilestonesFinder.new(params).execute milestones = child_milestones.select(:id, :title).group_by(&:title).map do |title, grouped| milestones_relation = Milestone.where(id: grouped.map(&:id)) @@ -28,13 +32,42 @@ class GlobalMilestone new(title, child_milestones) end - def self.states_count(projects) - relation = MilestonesFinder.new.execute(projects, state: 'all') - milestones_by_state_and_title = relation.reorder(nil).group(:state, :title).count + def self.states_count(projects, group = nil) + legacy_group_milestones_count = legacy_group_milestone_states_count(projects) + group_milestones_count = group_milestones_states_count(group) + + legacy_group_milestones_count.merge(group_milestones_count) do |k, legacy_group_milestones_count, group_milestones_count| + legacy_group_milestones_count + group_milestones_count + end + end + + def self.group_milestones_states_count(group) + return STATE_COUNT_HASH unless group + + params = { group_ids: [group.id], state: 'all', order: nil } + + relation = MilestonesFinder.new(params).execute + grouped_by_state = relation.group(:state).count + + { + opened: grouped_by_state['active'] || 0, + closed: grouped_by_state['closed'] || 0, + all: grouped_by_state.values.sum + } + end + + # Counts the legacy group milestones which must be grouped by title + def self.legacy_group_milestone_states_count(projects) + return STATE_COUNT_HASH unless projects + + params = { project_ids: projects.map(&:id), state: 'all', order: nil } + + relation = MilestonesFinder.new(params).execute + project_milestones_by_state_and_title = relation.group(:state, :title).count - opened = count_by_state(milestones_by_state_and_title, 'active') - closed = count_by_state(milestones_by_state_and_title, 'closed') - all = milestones_by_state_and_title.map { |(_, title), _| title }.uniq.count + opened = count_by_state(project_milestones_by_state_and_title, 'active') + closed = count_by_state(project_milestones_by_state_and_title, 'closed') + all = project_milestones_by_state_and_title.map { |(_, title), _| title }.uniq.count { opened: opened, diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb new file mode 100644 index 00000000000..3df60ddc950 --- /dev/null +++ b/app/models/gpg_key.rb @@ -0,0 +1,107 @@ +class GpgKey < ActiveRecord::Base + KEY_PREFIX = '-----BEGIN PGP PUBLIC KEY BLOCK-----'.freeze + KEY_SUFFIX = '-----END PGP PUBLIC KEY BLOCK-----'.freeze + + include ShaAttribute + + sha_attribute :primary_keyid + sha_attribute :fingerprint + + belongs_to :user + has_many :gpg_signatures + + validates :user, presence: true + + validates :key, + presence: true, + uniqueness: true, + format: { + with: /\A#{KEY_PREFIX}((?!#{KEY_PREFIX})(?!#{KEY_SUFFIX}).)+#{KEY_SUFFIX}\Z/m, + message: "is invalid. A valid public GPG key begins with '#{KEY_PREFIX}' and ends with '#{KEY_SUFFIX}'" + } + + validates :fingerprint, + presence: true, + uniqueness: true, + # only validate when the `key` is valid, as we don't want the user to show + # the error about the fingerprint + unless: -> { errors.has_key?(:key) } + + validates :primary_keyid, + presence: true, + uniqueness: true, + # only validate when the `key` is valid, as we don't want the user to show + # the error about the fingerprint + unless: -> { errors.has_key?(:key) } + + before_validation :extract_fingerprint, :extract_primary_keyid + after_commit :update_invalid_gpg_signatures, on: :create + after_commit :notify_user, on: :create + + def primary_keyid + super&.upcase + end + + def fingerprint + super&.upcase + end + + def key=(value) + super(value&.strip) + end + + def user_infos + @user_infos ||= Gitlab::Gpg.user_infos_from_key(key) + end + + def verified_user_infos + user_infos.select do |user_info| + user_info[:email] == user.email + end + end + + def emails_with_verified_status + user_infos.map do |user_info| + [ + user_info[:email], + user_info[:email] == user.email + ] + end.to_h + end + + def verified? + emails_with_verified_status.any? { |_email, verified| verified } + end + + def update_invalid_gpg_signatures + InvalidGpgSignatureUpdateWorker.perform_async(self.id) + end + + def revoke + GpgSignature.where(gpg_key: self, valid_signature: true).update_all( + gpg_key_id: nil, + valid_signature: false, + updated_at: Time.zone.now + ) + + destroy + end + + private + + def extract_fingerprint + # we can assume that the result only contains one item as the validation + # only allows one key + self.fingerprint = Gitlab::Gpg.fingerprints_from_key(key).first + end + + def extract_primary_keyid + # we can assume that the result only contains one item as the validation + # only allows one key + self.primary_keyid = Gitlab::Gpg.primary_keyids_from_key(key).first + end + + def notify_user + NotificationService.new.new_gpg_key(self) + end +end diff --git a/app/models/gpg_signature.rb b/app/models/gpg_signature.rb new file mode 100644 index 00000000000..1ac0e123ff1 --- /dev/null +++ b/app/models/gpg_signature.rb @@ -0,0 +1,21 @@ +class GpgSignature < ActiveRecord::Base + include ShaAttribute + + sha_attribute :commit_sha + sha_attribute :gpg_key_primary_keyid + + belongs_to :project + belongs_to :gpg_key + + validates :commit_sha, presence: true + validates :project_id, presence: true + validates :gpg_key_primary_keyid, presence: true + + def gpg_key_primary_keyid + super&.upcase + end + + def commit + project.commit(commit_sha) + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 0b93460d473..bd5735ed82e 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -2,13 +2,12 @@ require 'carrierwave/orm/activerecord' class Group < Namespace include Gitlab::ConfigHelper - include Gitlab::VisibilityLevel include AccessRequestable include Avatarable include Referable include SelectForProjectAuthorization - has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source + has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members has_many :users, through: :group_members has_many :owners, @@ -16,12 +15,14 @@ class Group < Namespace through: :group_members, source: :user - has_many :requesters, -> { where.not(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' + has_many :requesters, -> { where.not(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' # rubocop:disable Cop/ActiveRecordDependent - has_many :project_group_links, dependent: :destroy + has_many :milestones + has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :shared_projects, through: :project_group_links, source: :project - has_many :notification_settings, dependent: :destroy, as: :source + has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent has_many :labels, class_name: 'GroupLabel' + has_many :variables, class_name: 'Ci::GroupVariable' validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :visibility_level_allowed_by_projects @@ -31,7 +32,7 @@ class Group < Namespace validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } mount_uploader :avatar, AvatarUploader - has_many :uploads, as: :model, dependent: :destroy + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent after_create :post_create_hook after_destroy :post_destroy_hook @@ -101,10 +102,6 @@ class Group < Namespace full_name end - def visibility_level_field - :visibility_level - end - def visibility_level_allowed_by_projects allowed_by_projects = self.projects.where('visibility_level > ?', self.visibility_level).none? @@ -170,10 +167,14 @@ class Group < Namespace end def has_owner?(user) + return false unless user + members_with_parents.owners.where(user_id: user).any? end def has_master?(user) + return false unless user + members_with_parents.masters.where(user_id: user).any? end @@ -215,13 +216,19 @@ class Group < Namespace end def members_with_parents - GroupMember.non_request.where(source_id: ancestors.pluck(:id).push(id)) + GroupMember.active.where(source_id: ancestors.pluck(:id).push(id)).where.not(user_id: nil) end def users_with_parents User.where(id: members_with_parents.select(:user_id)) end + def users_with_descendants + members_with_descendants = GroupMember.non_request.where(source_id: descendants.pluck(:id).push(id)) + + User.where(id: members_with_descendants.select(:user_id)) + end + def max_member_access_for_user(user) return GroupMember::OWNER if user.admin? @@ -242,6 +249,14 @@ class Group < Namespace } end + def secret_variables_for(ref, project) + list_of_ids = [self] + ancestors + variables = Ci::GroupVariable.where(group: list_of_ids) + variables = variables.unprotected unless project.protected_for?(ref) + variables = variables.group_by(&:group_id) + list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten + end + protected def update_two_factor_requirement diff --git a/app/models/group_milestone.rb b/app/models/group_milestone.rb index 86d38e5468b..65249bd7bfc 100644 --- a/app/models/group_milestone.rb +++ b/app/models/group_milestone.rb @@ -16,4 +16,8 @@ class GroupMilestone < GlobalMilestone def issues_finder_params { group_id: group.id } end + + def is_legacy_group_milestone? + true + end end diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index ee6165fd32d..a8c424a6614 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -1,11 +1,20 @@ class ProjectHook < WebHook - belongs_to :project + TRIGGERS = { + push_hooks: :push_events, + tag_push_hooks: :tag_push_events, + issue_hooks: :issues_events, + confidential_issue_hooks: :confidential_issues_events, + note_hooks: :note_events, + merge_request_hooks: :merge_requests_events, + job_hooks: :job_events, + pipeline_hooks: :pipeline_events, + wiki_page_hooks: :wiki_page_events + }.freeze + + TRIGGERS.each do |trigger, event| + scope trigger, -> { where(event => true) } + end - scope :issue_hooks, -> { where(issues_events: true) } - scope :confidential_issue_hooks, -> { where(confidential_issues_events: true) } - scope :note_hooks, -> { where(note_events: true) } - scope :merge_request_hooks, -> { where(merge_requests_events: true) } - scope :job_hooks, -> { where(job_events: true) } - scope :pipeline_hooks, -> { where(pipeline_events: true) } - scope :wiki_page_hooks, -> { where(wiki_page_events: true) } + belongs_to :project + validates :project, presence: true end diff --git a/app/models/hooks/service_hook.rb b/app/models/hooks/service_hook.rb index 40e43c27f91..aef11514945 100644 --- a/app/models/hooks/service_hook.rb +++ b/app/models/hooks/service_hook.rb @@ -1,5 +1,6 @@ class ServiceHook < WebHook belongs_to :service + validates :service, presence: true def execute(data) WebHookService.new(self, data, 'service_hook').execute diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index 1584235ab00..180c479c41b 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -1,5 +1,13 @@ class SystemHook < WebHook - scope :repository_update_hooks, -> { where(repository_update_events: true) } + TRIGGERS = { + repository_update_hooks: :repository_update_events, + push_hooks: :push_events, + tag_push_hooks: :tag_push_events + }.freeze + + TRIGGERS.each do |trigger, event| + scope trigger, -> { where(event => true) } + end default_value_for :push_events, false default_value_for :repository_update_events, true diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 7503f3739c3..5a70e114f56 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -1,21 +1,7 @@ class WebHook < ActiveRecord::Base include Sortable - default_value_for :push_events, true - default_value_for :issues_events, false - default_value_for :confidential_issues_events, false - default_value_for :note_events, false - default_value_for :merge_requests_events, false - default_value_for :tag_push_events, false - default_value_for :job_events, false - default_value_for :pipeline_events, false - default_value_for :repository_update_events, false - default_value_for :enable_ssl_verification, true - - has_many :web_hook_logs, dependent: :destroy - - scope :push_hooks, -> { where(push_events: true) } - scope :tag_push_hooks, -> { where(tag_push_events: true) } + has_many :web_hook_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent validates :url, presence: true, url: true diff --git a/app/models/hooks/web_hook_log.rb b/app/models/hooks/web_hook_log.rb index d73cfcf630d..e72c125fb69 100644 --- a/app/models/hooks/web_hook_log.rb +++ b/app/models/hooks/web_hook_log.rb @@ -1,9 +1,9 @@ class WebHookLog < ActiveRecord::Base belongs_to :web_hook - serialize :request_headers, Hash # rubocop:disable Cop/ActiverecordSerialize - serialize :request_data, Hash # rubocop:disable Cop/ActiverecordSerialize - serialize :response_headers, Hash # rubocop:disable Cop/ActiverecordSerialize + serialize :request_headers, Hash # rubocop:disable Cop/ActiveRecordSerialize + serialize :request_data, Hash # rubocop:disable Cop/ActiveRecordSerialize + serialize :response_headers, Hash # rubocop:disable Cop/ActiveRecordSerialize validates :web_hook, presence: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 3a9a6dba601..1c948c8957e 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -10,6 +10,7 @@ class Issue < ActiveRecord::Base include FasterCacheKeys include RelativePositioning include IgnorableColumn + include CreatedAtFilterable ignore_column :position @@ -23,9 +24,14 @@ class Issue < ActiveRecord::Base belongs_to :project belongs_to :moved_to, class_name: 'Issue' - has_many :events, as: :target, dependent: :destroy + has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + has_many :merge_requests_closing_issues, + class_name: 'MergeRequestsClosingIssues', + dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + has_many :issue_assignees + has_many :assignees, class_name: "User", through: :issue_assignees has_many :issue_assignees has_many :assignees, class_name: "User", through: :issue_assignees @@ -45,8 +51,6 @@ class Issue < ActiveRecord::Base scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') } scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') } - scope :created_after, -> (datetime) { where("created_at >= ?", datetime) } - scope :preload_associations, -> { preload(:labels, project: :namespace) } after_save :expire_etag_cache @@ -58,15 +62,14 @@ class Issue < ActiveRecord::Base state_machine :state, initial: :opened do event :close do - transition [:reopened, :opened] => :closed + transition [:opened] => :closed end event :reopen do - transition closed: :reopened + transition closed: :opened end state :opened - state :reopened state :closed before_transition any => :closed do |issue| @@ -295,11 +298,7 @@ class Issue < ActiveRecord::Base end def expire_etag_cache - key = Gitlab::Routing.url_helpers.realtime_changes_namespace_project_issue_path( - project.namespace, - project, - self - ) + key = Gitlab::Routing.url_helpers.realtime_changes_project_issue_path(project, self) Gitlab::EtagCaching::Store.new.touch(key) end end diff --git a/app/models/label.rb b/app/models/label.rb index ed6a8411da9..674bb3f2720 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -15,9 +15,9 @@ class Label < ActiveRecord::Base default_value_for :color, DEFAULT_COLOR - has_many :lists, dependent: :destroy + has_many :lists, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :priorities, class_name: 'LabelPriority' - has_many :label_links, dependent: :destroy + has_many :label_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :issues, through: :label_links, source: :target, source_type: 'Issue' has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' diff --git a/app/models/legacy_diff_note.rb b/app/models/legacy_diff_note.rb index 2d5909ab25e..c36be956ff0 100644 --- a/app/models/legacy_diff_note.rb +++ b/app/models/legacy_diff_note.rb @@ -7,7 +7,7 @@ class LegacyDiffNote < Note include NoteOnDiff - serialize :st_diff # rubocop:disable Cop/ActiverecordSerialize + serialize :st_diff # rubocop:disable Cop/ActiveRecordSerialize validates :line_code, presence: true, line_code: true diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 7712d5783e0..b7cf96abe83 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -1,5 +1,5 @@ class LfsObject < ActiveRecord::Base - has_many :lfs_objects_projects, dependent: :destroy + has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :lfs_objects_projects validates :oid, presence: true, uniqueness: true diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index f581a25f093..81e0776e79c 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -5,6 +5,7 @@ class MergeRequest < ActiveRecord::Base include Referable include Sortable include IgnorableColumn + include CreatedAtFilterable ignore_column :position @@ -12,26 +13,25 @@ class MergeRequest < ActiveRecord::Base belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" - has_many :merge_request_diffs, dependent: :destroy + has_many :merge_request_diffs has_one :merge_request_diff, - -> { order('merge_request_diffs.id DESC') } + -> { order('merge_request_diffs.id DESC') }, inverse_of: :merge_request belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline" - has_many :events, as: :target, dependent: :destroy + has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + has_many :merge_requests_closing_issues, + class_name: 'MergeRequestsClosingIssues', + dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent belongs_to :assignee, class_name: "User" - serialize :merge_params, Hash # rubocop:disable Cop/ActiverecordSerialize + serialize :merge_params, Hash # rubocop:disable Cop/ActiveRecordSerialize after_create :ensure_merge_request_diff, unless: :importing? after_update :reload_diff_if_branch_changed - delegate :commits, :real_size, :commits_sha, :commits_count, - to: :merge_request_diff, prefix: nil - # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests attr_accessor :allow_broken @@ -42,23 +42,23 @@ class MergeRequest < ActiveRecord::Base state_machine :state, initial: :opened do event :close do - transition [:reopened, :opened] => :closed + transition [:opened] => :closed end event :mark_as_merged do - transition [:reopened, :opened, :locked] => :merged + transition [:opened, :locked] => :merged end event :reopen do - transition closed: :reopened + transition closed: :opened end event :lock_mr do - transition [:reopened, :opened] => :locked + transition [:opened] => :locked end event :unlock_mr do - transition locked: :reopened + transition locked: :opened end after_transition any => :locked do |merge_request, transition| @@ -72,7 +72,6 @@ class MergeRequest < ActiveRecord::Base end state :opened - state :reopened state :closed state :merged state :locked @@ -197,11 +196,19 @@ class MergeRequest < ActiveRecord::Base } end - # This method is needed for compatibility with issues to not mess view and other code + # These method are needed for compatibility with issues to not mess view and other code def assignees Array(assignee) end + def assignee_ids + Array(assignee_id) + end + + def assignee_ids=(ids) + write_attribute(:assignee_id, ids.last) + end + def assignee_or_author?(user) author_id == user.id || assignee_id == user.id end @@ -213,6 +220,36 @@ class MergeRequest < ActiveRecord::Base "#{project.to_reference(from, full: full)}#{reference}" end + def commits + if persisted? + merge_request_diff.commits + elsif compare_commits + compare_commits.reverse + else + [] + end + end + + def commits_count + if persisted? + merge_request_diff.commits_count + elsif compare_commits + compare_commits.size + else + 0 + end + end + + def commit_shas + if persisted? + merge_request_diff.commit_shas + elsif compare_commits + compare_commits.reverse.map(&:sha) + else + [] + end + end + def first_commit merge_request_diff ? merge_request_diff.first_commit : compare_commits.first end @@ -235,9 +272,7 @@ class MergeRequest < ActiveRecord::Base def diff_size # Calling `merge_request_diff.diffs.real_size` will also perform # highlighting, which we don't need here. - return real_size if merge_request_diff - - diffs.real_size + merge_request_diff&.real_size || diffs.real_size end def diff_base_commit @@ -332,7 +367,7 @@ class MergeRequest < ActiveRecord::Base errors.add :branch_conflict, "You can not use same project/branch for source and target" end - if opened? || reopened? + if opened? similar_mrs = self.target_project.merge_requests.where(source_branch: source_branch, target_branch: target_branch, source_project_id: source_project.try(:id)).opened similar_mrs = similar_mrs.where('id not in (?)', self.id) if self.id if similar_mrs.any? @@ -508,7 +543,7 @@ class MergeRequest < ActiveRecord::Base def related_notes # Fetch comments only from last 100 commits commits_for_notes_limit = 100 - commit_ids = commits.last(commits_for_notes_limit).map(&:id) + commit_ids = commit_shas.take(commits_for_notes_limit) Note.where( "(project_id = :target_project_id AND noteable_type = 'MergeRequest' AND noteable_id = :mr_id) OR" + @@ -560,7 +595,7 @@ class MergeRequest < ActiveRecord::Base # running `ReferenceExtractor` on each of them separately. # This optimization does not apply to issues from external sources. def cache_merge_request_closes_issues!(current_user) - return if project.has_external_issue_tracker? + return unless project.issues_enabled? transaction do self.merge_requests_closing_issues.delete_all @@ -771,6 +806,7 @@ class MergeRequest < ActiveRecord::Base "refs/heads/#{source_branch}", ref_path ) + update_column(:ref_fetched, true) end def ref_path @@ -778,7 +814,13 @@ class MergeRequest < ActiveRecord::Base end def ref_fetched? - project.repository.ref_exists?(ref_path) + super || + begin + computed_value = project.repository.ref_exists?(ref_path) + update_column(:ref_fetched, true) if computed_value + + computed_value + end end def ensure_ref_fetched @@ -824,15 +866,18 @@ class MergeRequest < ActiveRecord::Base return Ci::Pipeline.none unless source_project @all_pipelines ||= source_project.pipelines - .where(sha: all_commits_sha, ref: source_branch) + .where(sha: all_commit_shas, ref: source_branch) .order(id: :desc) end # Note that this could also return SHA from now dangling commits # - def all_commits_sha + def all_commit_shas if persisted? - merge_request_diffs.flat_map(&:commits_sha).uniq + column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).pluck('DISTINCT(sha)') + serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas) + + (column_shas + serialised_shas).uniq elsif compare_commits compare_commits.to_a.reverse.map(&:id) else diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index f1ee4d3f7a9..ec87aee9310 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -11,9 +11,10 @@ class MergeRequestDiff < ActiveRecord::Base belongs_to :merge_request has_many :merge_request_diff_files, -> { order(:merge_request_diff_id, :relative_order) } + has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) } - serialize :st_commits # rubocop:disable Cop/ActiverecordSerialize - serialize :st_diffs # rubocop:disable Cop/ActiverecordSerialize + serialize :st_commits # rubocop:disable Cop/ActiveRecordSerialize + serialize :st_diffs # rubocop:disable Cop/ActiveRecordSerialize state_machine :state, initial: :empty do state :collected @@ -47,14 +48,13 @@ class MergeRequestDiff < ActiveRecord::Base # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content - ensure_commits_sha + ensure_commit_shas save_commits - reload_commits save_diffs keep_around_commits end - def ensure_commits_sha + def ensure_commit_shas merge_request.fetch_ref self.start_commit_sha ||= merge_request.target_branch_sha self.head_commit_sha ||= merge_request.source_branch_sha @@ -66,7 +66,7 @@ class MergeRequestDiff < ActiveRecord::Base # created before version 8.4 that does not store head_commit_sha in separate db field. def head_commit_sha if persisted? && super.nil? - last_commit.try(:sha) + last_commit_sha else super end @@ -97,16 +97,11 @@ class MergeRequestDiff < ActiveRecord::Base end def commits - @commits ||= load_commits(st_commits) + @commits ||= load_commits end - def reload_commits - @commits = nil - commits - end - - def last_commit - commits.first + def last_commit_sha + commit_shas.first end def first_commit @@ -131,8 +126,12 @@ class MergeRequestDiff < ActiveRecord::Base project.commit(head_commit_sha) end - def commits_sha - st_commits.map { |commit| commit[:id] } + def commit_shas + if st_commits.present? + st_commits.map { |commit| commit[:id] } + else + merge_request_diff_commits.map(&:sha) + end end def diff_refs=(new_diff_refs) @@ -207,7 +206,11 @@ class MergeRequestDiff < ActiveRecord::Base end def commits_count - st_commits.count + if st_commits.present? + st_commits.size + else + merge_request_diff_commits.size + end end def utf8_st_diffs @@ -231,35 +234,23 @@ class MergeRequestDiff < ActiveRecord::Base raw.any? { |element| VALID_CLASSES.include?(element.class) } end - def dump_commits(commits) - commits.map(&:to_hash) - end - - def load_commits(array) - array.map { |hash| Commit.new(Gitlab::Git::Commit.new(hash), merge_request.source_project) } - end - - # Load all commits related to current merge request diff from repo - # and save it as array of hashes in st_commits db field - def save_commits - new_attributes = {} - - commits = compare.commits - - if commits.present? - commits = Commit.decorate(commits, merge_request.source_project).reverse - new_attributes[:st_commits] = dump_commits(commits) - end - - update_columns_serialized(new_attributes) - end - def create_merge_request_diff_files(diffs) rows = diffs.map.with_index do |diff, index| - diff.to_hash.merge( + diff_hash = diff.to_hash.merge( + binary: false, merge_request_diff_id: self.id, relative_order: index ) + + # Compatibility with old diffs created with Psych. + diff_hash.tap do |hash| + diff_text = hash[:diff] + + if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? + hash[:binary] = true + hash[:diff] = [diff_text].pack('m0') + end + end end Gitlab::Database.bulk_insert('merge_request_diff_files', rows) @@ -288,18 +279,22 @@ class MergeRequestDiff < ActiveRecord::Base st_diffs end elsif merge_request_diff_files.present? - merge_request_diff_files - .as_json(only: Gitlab::Git::Diff::SERIALIZE_KEYS) - .map(&:with_indifferent_access) + merge_request_diff_files.map(&:to_hash) end end - # Load diffs between branches related to current merge request diff from repo - # and save it as array of hashes in st_diffs db field + def load_commits + commits = st_commits.presence || merge_request_diff_commits + + commits.map do |commit| + Commit.new(Gitlab::Git::Commit.new(commit.to_hash), merge_request.source_project) + end + end + def save_diffs new_attributes = {} - if commits.size.zero? + if compare.commits.size.zero? new_attributes[:state] = :empty else diff_collection = compare.diffs(Commit.max_diff_options) @@ -319,7 +314,13 @@ class MergeRequestDiff < ActiveRecord::Base new_attributes[:state] = :overflow if diff_collection.overflow? end - update_columns_serialized(new_attributes) + update(new_attributes) + end + + def save_commits + MergeRequestDiffCommit.create_bulk(self.id, compare.commits.reverse) + + merge_request_diff_commits.reload end def repository @@ -332,29 +333,6 @@ class MergeRequestDiff < ActiveRecord::Base project.merge_base_commit(head_commit_sha, start_commit_sha).try(:sha) end - # - # #save or #update_attributes providing changes on serialized attributes do a lot of - # serialization and deserialization calls resulting in bad performance. - # Using #update_columns solves the problem with just one YAML.dump per serialized attribute that we provide. - # As a tradeoff we need to reload the current instance to properly manage time objects on those serialized - # attributes. So to keep the same behaviour as the attribute assignment we reload the instance. - # The difference is in the usage of - # #write_attribute= (#update_attributes) and #raw_write_attribute= (#update_columns) - # - # Ex: - # - # new_attributes[:st_commits].first.slice(:committed_date) - # => {:committed_date=>2014-02-27 11:01:38 +0200} - # YAML.load(YAML.dump(new_attributes[:st_commits].first.slice(:committed_date))) - # => {:committed_date=>2014-02-27 10:01:38 +0100} - # - def update_columns_serialized(new_attributes) - return unless new_attributes.any? - - update_columns(new_attributes.merge(updated_at: current_time_from_proper_timezone)) - reload - end - def keep_around_commits [repository, merge_request.source_project.repository].each do |repo| repo.keep_around(start_commit_sha) diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb new file mode 100644 index 00000000000..cafdbe11849 --- /dev/null +++ b/app/models/merge_request_diff_commit.rb @@ -0,0 +1,38 @@ +class MergeRequestDiffCommit < ActiveRecord::Base + include ShaAttribute + + belongs_to :merge_request_diff + + sha_attribute :sha + alias_attribute :id, :sha + + def self.create_bulk(merge_request_diff_id, commits) + sha_attribute = Gitlab::Database::ShaAttribute.new + + rows = commits.map.with_index do |commit, index| + # See #parent_ids. + commit_hash = commit.to_hash.except(:parent_ids) + sha = commit_hash.delete(:id) + + commit_hash.merge( + merge_request_diff_id: merge_request_diff_id, + relative_order: index, + sha: sha_attribute.type_cast_for_database(sha) + ) + end + + Gitlab::Database.bulk_insert(self.table_name, rows) + end + + def to_hash + Gitlab::Git::Commit::SERIALIZE_KEYS.each_with_object({}) do |key, hash| + hash[key] = public_send(key) + end + end + + # We don't save these, because they would need a table or a serialised + # field. They aren't used anywhere, so just pretend the commit has no parents. + def parent_ids + [] + end +end diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index 598ebd4d829..1199ff5af22 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -8,4 +8,14 @@ class MergeRequestDiffFile < ActiveRecord::Base encode_utf8(diff) if diff.respond_to?(:encoding) end + + def diff + binary? ? super.unpack('m0').first : super + end + + def to_hash + keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff] + + as_json(only: keys).merge(diff: diff).with_indifferent_access + end end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index d2e2749f70d..48d00764965 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -18,17 +18,32 @@ class Milestone < ActiveRecord::Base cache_markdown_field :description belongs_to :project + belongs_to :group + has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests - has_many :events, as: :target, dependent: :destroy + has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + scope :of_projects, ->(ids) { where(project_id: ids) } + scope :of_groups, ->(ids) { where(group_id: ids) } scope :active, -> { with_state(:active) } scope :closed, -> { with_state(:closed) } - scope :of_projects, ->(ids) { where(project_id: ids) } + scope :for_projects, -> { where(group: nil).includes(:project) } + + scope :for_projects_and_groups, -> (project_ids, group_ids) do + conditions = [] + conditions << arel_table[:project_id].in(project_ids) if project_ids.compact.any? + conditions << arel_table[:group_id].in(group_ids) if group_ids.compact.any? + + where(conditions.reduce(:or)) + end + + validates :group, presence: true, unless: :project + validates :project, presence: true, unless: :group - validates :title, presence: true, uniqueness: { scope: :project_id } - validates :project, presence: true + validate :uniqueness_of_title, if: :title_changed? + validate :milestone_type_check validate :start_date_should_be_less_than_due_date, if: proc { |m| m.start_date.present? && m.due_date.present? } strip_attributes :title @@ -63,6 +78,14 @@ class Milestone < ActiveRecord::Base where(t[:title].matches(pattern).or(t[:description].matches(pattern))) end + + def filter_by_state(milestones, state) + case state + when 'closed' then milestones.closed + when 'all' then milestones + else milestones.active + end + end end def self.reference_prefix @@ -138,6 +161,8 @@ class Milestone < ActiveRecord::Base # Milestone.first.to_reference(same_namespace_project) # => "gitlab-ce%1" # def to_reference(from_project = nil, format: :iid, full: false) + return if is_group_milestone? + format_reference = milestone_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" @@ -152,6 +177,10 @@ class Milestone < ActiveRecord::Base id end + def for_display + self + end + def can_be_closed? active? && issues.opened.count.zero? end @@ -164,8 +193,45 @@ class Milestone < ActiveRecord::Base write_attribute(:title, sanitize_title(value)) if value.present? end + def safe_title + title.to_slug.normalize.to_s + end + + def parent + group || project + end + + def is_group_milestone? + group_id.present? + end + + def is_project_milestone? + project_id.present? + end + private + # Milestone titles must be unique across project milestones and group milestones + def uniqueness_of_title + if project + relation = Milestone.for_projects_and_groups([project_id], [project.group&.id]) + elsif group + project_ids = group.projects.map(&:id) + relation = Milestone.for_projects_and_groups(project_ids, [group.id]) + end + + title_exists = relation.find_by_title(title) + errors.add(:title, "already being used for another group or project milestone.") if title_exists + end + + # Milestone should be either a project milestone or a group milestone + def milestone_type_check + if group_id && project_id + field = project_id_changed? ? :project_id : :group_id + errors.add(field, "milestone should belong either to a project or a group.") + end + end + def milestone_format_reference(format = :iid) raise ArgumentError, 'Unknown format' unless [:iid, :name].include?(format) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 583d4fb5244..0bb04194bdb 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -1,10 +1,11 @@ class Namespace < ActiveRecord::Base - acts_as_paranoid + acts_as_paranoid without_default_scope: true include CacheMarkdownField include Sortable include Gitlab::ShellAdapter include Gitlab::CurrentSettings + include Gitlab::VisibilityLevel include Routable include AfterCommitQueue @@ -15,13 +16,13 @@ class Namespace < ActiveRecord::Base cache_markdown_field :description, pipeline: :description - has_many :projects, dependent: :destroy + has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics belongs_to :owner, class_name: "User" belongs_to :parent, class_name: "Namespace" has_many :children, class_name: "Namespace", foreign_key: :parent_id - has_one :chat_team, dependent: :destroy + has_one :chat_team, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent validates :owner, presence: true, unless: ->(n) { n.type == "Group" } validates :name, @@ -105,6 +106,10 @@ class Namespace < ActiveRecord::Base end end + def visibility_level_field + :visibility_level + end + def to_param full_path end @@ -219,6 +224,12 @@ class Namespace < ActiveRecord::Base parent.present? end + def soft_delete_without_removing_associations + # We can't use paranoia's `#destroy` since this will hard-delete projects. + # Project uses `pending_delete` instead of the acts_as_paranoia gem. + self.deleted_at = Time.now + end + private def repository_storage_paths diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb index 59737bb6085..2bc00a082df 100644 --- a/app/models/network/graph.rb +++ b/app/models/network/graph.rb @@ -113,7 +113,7 @@ module Network opts[:ref] = @commit.id if @filter_ref - @repo.find_commits(opts) + Gitlab::Git::Commit.find_all(@repo.raw_repository, opts) end def commits_sort_by_ref diff --git a/app/models/note.rb b/app/models/note.rb index ca6999427c0..d0e3bc0bfed 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -46,8 +46,8 @@ class Note < ActiveRecord::Base belongs_to :updated_by, class_name: "User" belongs_to :last_edited_by, class_name: 'User' - has_many :todos, dependent: :destroy - has_many :events, as: :target, dependent: :destroy + has_many :todos, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :system_note_metadata delegate :gfm_reference, :local_reference, to: :noteable @@ -190,7 +190,7 @@ class Note < ActiveRecord::Base # override to return commits, which are not active record def noteable if for_commit? - project.commit(commit_id) + @commit ||= project.commit(commit_id) else super end @@ -330,8 +330,7 @@ class Note < ActiveRecord::Base def expire_etag_cache return unless for_issue? - key = Gitlab::Routing.url_helpers.namespace_project_noteable_notes_path( - noteable.project.namespace, + key = Gitlab::Routing.url_helpers.project_noteable_notes_path( noteable.project, target_type: noteable_type.underscore, target_id: noteable.id diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index b0df7aeb323..81844b1e2ca 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -19,7 +19,7 @@ class NotificationSetting < ActiveRecord::Base # pending delete). # scope :for_projects, -> do - includes(:project).references(:projects).where(source_type: 'Project').where.not(projects: { id: nil }) + includes(:project).references(:projects).where(source_type: 'Project').where.not(projects: { id: nil, pending_delete: true }) end EMAIL_EVENTS = [ diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 6e13f9b2089..654be927ed8 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -3,7 +3,7 @@ class PersonalAccessToken < ActiveRecord::Base include TokenAuthenticatable add_authentication_token_field :token - serialize :scopes, Array # rubocop:disable Cop/ActiverecordSerialize + serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize belongs_to :user diff --git a/app/models/project.rb b/app/models/project.rb index 2c2685875f8..d827bfaa806 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -59,6 +59,7 @@ class Project < ActiveRecord::Base update_column(:last_repository_updated_at, self.created_at) end + before_destroy :remove_private_deploy_keys after_destroy :remove_pages # update visibility_level of forks @@ -80,96 +81,108 @@ class Project < ActiveRecord::Base belongs_to :namespace has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event' - has_many :boards, before_add: :validate_board_limit, dependent: :destroy + has_many :boards, before_add: :validate_board_limit # Project services - has_one :campfire_service, dependent: :destroy - has_one :drone_ci_service, dependent: :destroy - has_one :emails_on_push_service, dependent: :destroy - has_one :pipelines_email_service, dependent: :destroy - has_one :irker_service, dependent: :destroy - has_one :pivotaltracker_service, dependent: :destroy - has_one :hipchat_service, dependent: :destroy - has_one :flowdock_service, dependent: :destroy - has_one :assembla_service, dependent: :destroy - has_one :asana_service, dependent: :destroy - has_one :gemnasium_service, dependent: :destroy - has_one :mattermost_slash_commands_service, dependent: :destroy - has_one :mattermost_service, dependent: :destroy - has_one :slack_slash_commands_service, dependent: :destroy - has_one :slack_service, dependent: :destroy - has_one :buildkite_service, dependent: :destroy - has_one :bamboo_service, dependent: :destroy - has_one :teamcity_service, dependent: :destroy - has_one :pushover_service, dependent: :destroy - has_one :jira_service, dependent: :destroy - has_one :redmine_service, dependent: :destroy - has_one :custom_issue_tracker_service, dependent: :destroy - has_one :bugzilla_service, dependent: :destroy - has_one :gitlab_issue_tracker_service, dependent: :destroy, inverse_of: :project - has_one :external_wiki_service, dependent: :destroy - has_one :kubernetes_service, dependent: :destroy, inverse_of: :project - has_one :prometheus_service, dependent: :destroy, inverse_of: :project - has_one :mock_ci_service, dependent: :destroy - has_one :mock_deployment_service, dependent: :destroy - has_one :mock_monitoring_service, dependent: :destroy - has_one :microsoft_teams_service, dependent: :destroy - - has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id" + has_one :campfire_service + has_one :drone_ci_service + has_one :emails_on_push_service + has_one :pipelines_email_service + has_one :irker_service + has_one :pivotaltracker_service + has_one :hipchat_service + has_one :flowdock_service + has_one :assembla_service + has_one :asana_service + has_one :gemnasium_service + has_one :mattermost_slash_commands_service + has_one :mattermost_service + has_one :slack_slash_commands_service + has_one :slack_service + has_one :buildkite_service + has_one :bamboo_service + has_one :teamcity_service + has_one :pushover_service + has_one :jira_service + has_one :redmine_service + has_one :custom_issue_tracker_service + has_one :bugzilla_service + has_one :gitlab_issue_tracker_service, inverse_of: :project + has_one :external_wiki_service + has_one :kubernetes_service, inverse_of: :project + has_one :prometheus_service, inverse_of: :project + has_one :mock_ci_service + has_one :mock_deployment_service + has_one :mock_monitoring_service + has_one :microsoft_teams_service + + has_one :forked_project_link, foreign_key: "forked_to_project_id" has_one :forked_from_project, through: :forked_project_link has_many :forked_project_links, foreign_key: "forked_from_project_id" has_many :forks, through: :forked_project_links, source: :forked_to_project # Merge Requests for target project should be removed with it - has_many :merge_requests, dependent: :destroy, foreign_key: 'target_project_id' - has_many :issues, dependent: :destroy - has_many :labels, dependent: :destroy, class_name: 'ProjectLabel' - has_many :services, dependent: :destroy - has_many :events, dependent: :destroy - has_many :milestones, dependent: :destroy - has_many :notes, dependent: :destroy - has_many :snippets, dependent: :destroy, class_name: 'ProjectSnippet' - has_many :hooks, dependent: :destroy, class_name: 'ProjectHook' - has_many :protected_branches, dependent: :destroy - has_many :protected_tags, dependent: :destroy + has_many :merge_requests, foreign_key: 'target_project_id' + has_many :issues + has_many :labels, class_name: 'ProjectLabel' + has_many :services + has_many :events + has_many :milestones + has_many :notes + has_many :snippets, class_name: 'ProjectSnippet' + has_many :hooks, class_name: 'ProjectHook' + has_many :protected_branches + has_many :protected_tags has_many :project_authorizations has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' - has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source + has_many :project_members, -> { where(requested_at: nil) }, + as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + alias_method :members, :project_members has_many :users, through: :project_members - has_many :requesters, -> { where.not(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'ProjectMember' + has_many :requesters, -> { where.not(requested_at: nil) }, + as: :source, class_name: 'ProjectMember', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - has_many :deploy_keys_projects, dependent: :destroy + has_many :deploy_keys_projects has_many :deploy_keys, through: :deploy_keys_projects - has_many :users_star_projects, dependent: :destroy + has_many :users_star_projects has_many :starrers, through: :users_star_projects, source: :user - has_many :releases, dependent: :destroy - has_many :lfs_objects_projects, dependent: :destroy + has_many :releases + has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :lfs_objects, through: :lfs_objects_projects - has_many :project_group_links, dependent: :destroy + has_many :project_group_links has_many :invited_groups, through: :project_group_links, source: :group - has_many :pages_domains, dependent: :destroy - has_many :todos, dependent: :destroy - has_many :notification_settings, dependent: :destroy, as: :source - - has_one :import_data, dependent: :delete, class_name: 'ProjectImportData' - has_one :project_feature, dependent: :destroy - has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete - has_many :container_repositories, dependent: :destroy - - has_many :commit_statuses, dependent: :destroy - has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline' - has_many :builds, class_name: 'Ci::Build' # the builds are created from the commit_statuses - has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject' + has_many :pages_domains + has_many :todos + has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + has_one :import_data, class_name: 'ProjectImportData' + has_one :project_feature + has_one :statistics, class_name: 'ProjectStatistics' + + # Container repositories need to remove data from the container registry, + # which is not managed by the DB. Hence we're still using dependent: :destroy + # here. + has_many :container_repositories, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + + has_many :commit_statuses + has_many :pipelines, class_name: 'Ci::Pipeline' + + # Ci::Build objects store data on the file system such as artifact files and + # build traces. Currently there's no efficient way of removing this data in + # bulk that doesn't involve loading the rows into memory. As a result we're + # still using `dependent: :destroy` here. + has_many :builds, class_name: 'Ci::Build', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :runner_projects, class_name: 'Ci::RunnerProject' has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' - has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger' - has_many :environments, dependent: :destroy - has_many :deployments, dependent: :destroy - has_many :pipeline_schedules, dependent: :destroy, class_name: 'Ci::PipelineSchedule' + has_many :triggers, class_name: 'Ci::Trigger' + has_many :environments + has_many :deployments + has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' @@ -186,6 +199,11 @@ class Project < ActiveRecord::Base # Validations validates :creator, presence: true, on: :create validates :description, length: { maximum: 2000 }, allow_blank: true + validates :ci_config_path, + format: { without: /\.{2}/, + message: 'cannot include directory traversal.' }, + length: { maximum: 255 }, + allow_blank: true validates :name, presence: true, length: { maximum: 255 }, @@ -219,12 +237,11 @@ class Project < ActiveRecord::Base before_save :ensure_runners_token mount_uploader :avatar, AvatarUploader - has_many :uploads, as: :model, dependent: :destroy + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Scopes - default_scope { where(pending_delete: false) } - - scope :with_deleted, -> { unscope(where: :pending_delete) } + scope :pending_delete, -> { where(pending_delete: true) } + scope :without_deleted, -> { where(pending_delete: false) } scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) } scope :sorted_by_stars, -> { reorder('projects.star_count DESC') } @@ -350,7 +367,19 @@ class Project < ActiveRecord::Base project.run_after_commit { add_import_job } end - after_transition started: :finished, do: :reset_cache_and_import_attrs + after_transition started: :finished do |project, _| + project.reset_cache_and_import_attrs + + if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists? + project.run_after_commit do + begin + Projects::HousekeepingService.new(project).execute + rescue Projects::HousekeepingService::LeaseTaken => e + Rails.logger.info("Could not perform housekeeping for project #{project.path_with_namespace} (#{project.id}): #{e}") + end + end + end + end end class << self @@ -457,7 +486,9 @@ class Project < ActiveRecord::Base end def has_container_registry_tags? - container_repositories.to_a.any?(&:has_tags?) || + return @images if defined?(@images) + + @images = container_repositories.to_a.any?(&:has_tags?) || has_root_container_repository_tags? end @@ -510,10 +541,16 @@ class Project < ActiveRecord::Base remove_import_data end + # This method is overriden in EE::Project model def remove_import_data import_data&.destroy end + def ci_config_path=(value) + # Strip all leading slashes so that //foo -> foo + super(value&.sub(%r{\A/+}, '')&.delete("\0")) + end + def import_url=(value) return super(value) unless Gitlab::UrlSanitizer.valid?(value) @@ -668,7 +705,7 @@ class Project < ActiveRecord::Base end def web_url - Gitlab::Routing.url_helpers.namespace_project_url(self.namespace, self) + Gitlab::Routing.url_helpers.project_url(self) end def new_issue_address(author) @@ -689,7 +726,7 @@ class Project < ActiveRecord::Base end def last_activity_date - last_activity_at || updated_at + last_repository_updated_at || last_activity_at || updated_at end def project_id @@ -697,9 +734,11 @@ class Project < ActiveRecord::Base end def get_issue(issue_id, current_user) - if default_issues_tracker? - IssuesFinder.new(current_user, project_id: id).find_by(iid: issue_id) - else + issue = IssuesFinder.new(current_user, project_id: id).find_by(iid: issue_id) if issues_enabled? + + if issue + issue + elsif external_issue_tracker ExternalIssue.new(issue_id, self) end end @@ -720,8 +759,8 @@ class Project < ActiveRecord::Base end end - def issue_reference_pattern - issues_tracker.reference_pattern + def external_issue_reference_pattern + external_issue_tracker.class.reference_pattern(only_long: issues_enabled?) end def default_issues_tracker? @@ -766,10 +805,12 @@ class Project < ActiveRecord::Base update_column(:has_external_wiki, services.external_wikis.any?) end - def find_or_initialize_services + def find_or_initialize_services(exceptions: []) services_templates = Service.where(template: true) - Service.available_services_names.map do |service_name| + available_services_names = Service.available_services_names - exceptions + + available_services_names.map do |service_name| service = find_service(services, service_name) if service @@ -844,7 +885,7 @@ class Project < ActiveRecord::Base def avatar_url(**args) # We use avatar_path instead of overriding avatar_url because of carrierwave. # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - avatar_path(args) || (Gitlab::Routing.url_helpers.namespace_project_avatar_url(namespace, self) if avatar_in_git) + avatar_path(args) || (Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git) end # For compatibility with old code @@ -940,8 +981,6 @@ class Project < ActiveRecord::Base Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - expire_caches_before_rename(old_path_with_namespace) - if has_container_registry_tags? Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" @@ -949,6 +988,8 @@ class Project < ActiveRecord::Base raise StandardError.new('Project cannot be renamed, because images are present in its container registry') end + expire_caches_before_rename(old_path_with_namespace) + if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository @@ -956,6 +997,7 @@ class Project < ActiveRecord::Base begin gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") send_move_instructions(old_path_with_namespace) + expires_full_path_cache @old_path_with_namespace = old_path_with_namespace @@ -1007,7 +1049,8 @@ class Project < ActiveRecord::Base namespace: namespace.name, visibility_level: visibility_level, path_with_namespace: path_with_namespace, - default_branch: default_branch + default_branch: default_branch, + ci_config_path: ci_config_path } # Backward compatibility @@ -1066,19 +1109,23 @@ class Project < ActiveRecord::Base merge_requests.where(source_project_id: self.id) end - def create_repository + def create_repository(force: false) # Forked import is handled asynchronously - unless forked? - if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) - repository.after_create - true - else - errors.add(:base, 'Failed to create repository via gitlab-shell') - false - end + return if forked? && !force + + if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) + repository.after_create + true + else + errors.add(:base, 'Failed to create repository via gitlab-shell') + false end end + def ensure_repository + create_repository(force: true) unless repository_exists? + end + def repository_exists? !!repository.exists? end @@ -1217,7 +1264,13 @@ class Project < ActiveRecord::Base File.join(pages_path, 'public') end + def remove_private_deploy_keys + deploy_keys.where(public: false).delete_all + end + def remove_pages + ::Projects::UpdatePagesConfigurationService.new(self).execute + # 1. We rename pages to temporary directory # 2. We wait 5 minutes, due to NFS caching # 3. We asynchronously remove pages with force @@ -1303,7 +1356,8 @@ class Project < ActiveRecord::Base variables end - def secret_variables_for(ref) + def secret_variables_for(ref:, environment: nil) + # EE would use the environment if protected_for?(ref) variables else @@ -1336,15 +1390,15 @@ class Project < ActiveRecord::Base end def pushes_since_gc - Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i } + Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i } end def increment_pushes_since_gc - Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) } + Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) } end def reset_pushes_since_gc - Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) } + Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) } end def route_map_for(commit_sha) @@ -1407,7 +1461,7 @@ class Project < ActiveRecord::Base from && self != from end - def pushes_since_gc_redis_key + def pushes_since_gc_redis_shared_state_key "projects/#{id}/pushes_since_gc" end @@ -1441,7 +1495,7 @@ class Project < ActiveRecord::Base def pending_delete_twin return false unless path - Project.unscoped.where(pending_delete: true).find_by_full_path(path_with_namespace) + Project.pending_delete.find_by_full_path(path_with_namespace) end ## diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index dde2a11440d..c8fabb16dc1 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -51,8 +51,11 @@ class ProjectFeature < ActiveRecord::Base default_value_for :repository_access_level, value: ENABLED, allows_nil: false def feature_available?(feature, user) - access_level = public_send(ProjectFeature.access_level_attribute(feature)) - get_permission(user, access_level) + get_permission(user, access_level(feature)) + end + + def access_level(feature) + public_send(ProjectFeature.access_level_attribute(feature)) end def builds_enabled? @@ -90,7 +93,7 @@ class ProjectFeature < ActiveRecord::Base when DISABLED false when PRIVATE - user && (project.team.member?(user) || user.admin?) + user && (project.team.member?(user) || user.full_private_access?) when ENABLED true else diff --git a/app/models/project_import_data.rb b/app/models/project_import_data.rb index e3cafd4d1c6..37730474324 100644 --- a/app/models/project_import_data.rb +++ b/app/models/project_import_data.rb @@ -10,7 +10,7 @@ class ProjectImportData < ActiveRecord::Base insecure_mode: true, algorithm: 'aes-256-cbc' - serialize :data, JSON # rubocop:disable Cop/ActiverecordSerialize + serialize :data, JSON # rubocop:disable Cop/ActiveRecordSerialize validates :project, presence: true diff --git a/app/models/project_services/drone_ci_service.rb b/app/models/project_services/drone_ci_service.rb index f6cade9c290..c93f1632652 100644 --- a/app/models/project_services/drone_ci_service.rb +++ b/app/models/project_services/drone_ci_service.rb @@ -114,7 +114,7 @@ class DroneCiService < CiService end def merge_request_valid?(data) - %w(opened reopened).include?(data[:object_attributes][:state]) && + data[:object_attributes][:state] == 'opened' && data[:object_attributes][:merge_status] == 'unchecked' end end diff --git a/app/models/project_services/gitlab_issue_tracker_service.rb b/app/models/project_services/gitlab_issue_tracker_service.rb index ad4eb9536e1..88c428b4aae 100644 --- a/app/models/project_services/gitlab_issue_tracker_service.rb +++ b/app/models/project_services/gitlab_issue_tracker_service.rb @@ -1,5 +1,5 @@ class GitlabIssueTrackerService < IssueTrackerService - include Gitlab::Routing.url_helpers + include Gitlab::Routing validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? @@ -12,26 +12,26 @@ class GitlabIssueTrackerService < IssueTrackerService end def project_url - namespace_project_issues_url(project.namespace, project) + project_issues_url(project) end def new_issue_url - new_namespace_project_issue_url(namespace_id: project.namespace, project_id: project) + new_project_issue_url(project) end def issue_url(iid) - namespace_project_issue_url(namespace_id: project.namespace, project_id: project, id: iid) + project_issue_url(project, id: iid) end - def project_path - namespace_project_issues_path(project.namespace, project) + def issue_tracker_path + project_issues_path(project) end def new_issue_path - new_namespace_project_issue_path(namespace_id: project.namespace, project_id: project) + new_project_issue_path(project) end def issue_path(iid) - namespace_project_issue_path(namespace_id: project.namespace, project_id: project, id: iid) + project_issue_path(project, id: iid) end end diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb index ff138b9066d..31984c5d7ed 100644 --- a/app/models/project_services/issue_tracker_service.rb +++ b/app/models/project_services/issue_tracker_service.rb @@ -5,8 +5,15 @@ class IssueTrackerService < Service # Pattern used to extract links from comments # Override this method on services that uses different patterns - def reference_pattern - @reference_pattern ||= %r{(\b[A-Z][A-Z0-9_]+-|#{Issue.reference_prefix})(?<issue>\d+)} + # This pattern does not support cross-project references + # The other code assumes that this pattern is a superset of all + # overriden patterns. See ReferenceRegexes::EXTERNAL_PATTERN + def self.reference_pattern(only_long: false) + if only_long + %r{(\b[A-Z][A-Z0-9_]+-)(?<issue>\d+)} + else + %r{(\b[A-Z][A-Z0-9_]+-|#{Issue.reference_prefix})(?<issue>\d+)} + end end def default? @@ -17,7 +24,7 @@ class IssueTrackerService < Service self.issues_url.gsub(':id', iid.to_s) end - def project_path + def issue_tracker_path project_url end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 2450fb43212..2aa19443198 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -1,12 +1,10 @@ class JiraService < IssueTrackerService - include Gitlab::Routing.url_helpers + include Gitlab::Routing validates :url, url: true, presence: true, if: :activated? validates :api_url, url: true, allow_blank: true - validates :project_key, presence: true, if: :activated? - prop_accessor :username, :password, :url, :api_url, :project_key, - :jira_issue_transition_id, :title, :description + prop_accessor :username, :password, :url, :api_url, :jira_issue_transition_id, :title, :description before_update :reset_password @@ -18,7 +16,7 @@ class JiraService < IssueTrackerService end # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 - def reference_pattern + def self.reference_pattern(only_long: true) @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)} end @@ -54,10 +52,6 @@ class JiraService < IssueTrackerService @client ||= JIRA::Client.new(options) end - def jira_project - @jira_project ||= jira_request { client.Project.find(project_key) } - end - def help "You need to configure JIRA before enabling this service. For more details read the @@ -88,18 +82,12 @@ class JiraService < IssueTrackerService [ { type: 'text', name: 'url', title: 'Web URL', placeholder: 'https://jira.example.com', required: true }, { type: 'text', name: 'api_url', title: 'JIRA API URL', placeholder: 'If different from Web URL' }, - { type: 'text', name: 'project_key', placeholder: 'Project Key', required: true }, { type: 'text', name: 'username', placeholder: '', required: true }, { type: 'password', name: 'password', placeholder: '', required: true }, - { type: 'text', name: 'jira_issue_transition_id', placeholder: '' } + { type: 'text', name: 'jira_issue_transition_id', title: 'Transition ID', placeholder: '' } ] end - # URLs to redirect from Gitlab issues pages to jira issue tracker - def project_url - "#{url}/issues/?jql=project=#{project_key}" - end - def issues_url "#{url}/browse/:id" end @@ -152,8 +140,8 @@ class JiraService < IssueTrackerService url: resource_url(user_path(author)) }, project: { - name: self.project.path_with_namespace, - url: resource_url(namespace_project_path(project.namespace, self.project)) + name: project.path_with_namespace, + url: resource_url(namespace_project_path(project.namespace, project)) # rubocop:disable Cop/ProjectPathHelper }, entity: { name: noteable_type.humanize.downcase, @@ -172,7 +160,10 @@ class JiraService < IssueTrackerService def test(_) result = test_settings - { success: result.present?, result: result } + success = result.present? + result = @error if @error && !success + + { success: success, result: result } end # JIRA does not need test data. @@ -184,7 +175,7 @@ class JiraService < IssueTrackerService def test_settings return unless client_url.present? # Test settings by getting the project - jira_request { jira_project.present? } + jira_request { client.ServerInfo.all.attrs } end private @@ -300,7 +291,8 @@ class JiraService < IssueTrackerService yield rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, URI::InvalidURIError, JIRA::HTTPError, OpenSSL::SSL::SSLError => e - Rails.logger.info "#{self.class.name} Send message ERROR: #{client_url} - #{e.message}" + @error = e.message + Rails.logger.info "#{self.class.name} Send message ERROR: #{client_url} - #{@error}" nil end diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 48e7802c557..dee99bbb859 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -59,21 +59,21 @@ class KubernetesService < DeploymentService def fields [ { type: 'text', - name: 'namespace', - title: 'Kubernetes namespace', - placeholder: namespace_placeholder }, - { type: 'text', name: 'api_url', title: 'API URL', placeholder: 'Kubernetes API URL, like https://kube.example.com/' }, - { type: 'text', - name: 'token', - title: 'Service token', - placeholder: 'Service token' }, { type: 'textarea', name: 'ca_pem', - title: 'Custom CA bundle', - placeholder: 'Certificate Authority bundle (PEM format)' } + title: 'CA Certificate', + placeholder: 'Certificate Authority bundle (PEM format)' }, + { type: 'text', + name: 'namespace', + title: 'Project namespace (optional/unique)', + placeholder: namespace_placeholder }, + { type: 'text', + name: 'token', + title: 'Token', + placeholder: 'Service token' } ] end @@ -96,10 +96,13 @@ class KubernetesService < DeploymentService end def predefined_variables + config = YAML.dump(kubeconfig) + variables = [ { key: 'KUBE_URL', value: api_url, public: true }, { key: 'KUBE_TOKEN', value: token, public: false }, - { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true } + { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true }, + { key: 'KUBECONFIG', value: config, public: false, file: true } ] if ca_pem.present? @@ -135,6 +138,14 @@ class KubernetesService < DeploymentService private + def kubeconfig + to_kubeconfig( + url: api_url, + namespace: actual_namespace, + token: token, + ca_pem: ca_pem) + end + def namespace_placeholder default_namespace || TEMPLATE_PLACEHOLDER end diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 110b8bc209b..217f753f05f 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -28,17 +28,6 @@ class PrometheusService < MonitoringService 'Prometheus monitoring' end - def help - <<-MD.strip_heredoc - Retrieves the Kubernetes node metrics `container_cpu_usage_seconds_total` - and `container_memory_usage_bytes` from the configured Prometheus server. - - If you are not using [Auto-Deploy](https://docs.gitlab.com/ee/ci/autodeploy/index.html) - or have set up your own Prometheus server, an `environment` label is required on each metric to - [identify the Environment](https://docs.gitlab.com/ce/user/project/integrations/prometheus.html#metrics-and-labels). - MD - end - def self.to_param 'prometheus' end @@ -50,6 +39,7 @@ class PrometheusService < MonitoringService name: 'api_url', title: 'API URL', placeholder: 'Prometheus API Base URL, like http://prometheus.example.com/', + help: 'By default, Prometheus listens on ‘http://localhost:9090’. It’s not recommended to change the default address and port as this might affect or conflict with other services running on the GitLab server.', required: true } ] @@ -65,23 +55,34 @@ class PrometheusService < MonitoringService end def environment_metrics(environment) - with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &:itself) + with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics)) end def deployment_metrics(deployment) - metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &:itself) - metrics&.merge(deployment_time: created_at.to_i) || {} + metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &method(:rename_data_to_metrics)) + metrics&.merge(deployment_time: deployment.created_at.to_i) || {} + end + + def additional_environment_metrics(environment) + with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsEnvironmentQuery.name, environment.id, &:itself) + end + + def additional_deployment_metrics(deployment) + with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery.name, deployment.id, &:itself) + end + + def matched_metrics + with_reactive_cache(Gitlab::Prometheus::Queries::MatchedMetricsQuery.name, &:itself) end # Cache metrics for specific environment def calculate_reactive_cache(query_class_name, *args) return unless active? && project && !project.pending_delete? - metrics = Kernel.const_get(query_class_name).new(client).query(*args) - + data = Kernel.const_get(query_class_name).new(client).query(*args) { success: true, - metrics: metrics, + data: data, last_update: Time.now.utc } rescue Gitlab::PrometheusError => err @@ -91,4 +92,11 @@ class PrometheusService < MonitoringService def client @prometheus ||= Gitlab::PrometheusClient.new(api_url: api_url) end + + private + + def rename_data_to_metrics(metrics) + metrics[:metrics] = metrics.delete :data + metrics + end end diff --git a/app/models/project_services/slash_commands_service.rb b/app/models/project_services/slash_commands_service.rb index 4592cb747a0..eb4da68bb7e 100644 --- a/app/models/project_services/slash_commands_service.rb +++ b/app/models/project_services/slash_commands_service.rb @@ -5,7 +5,7 @@ class SlashCommandsService < Service prop_accessor :token - has_many :chat_names, foreign_key: :service_id, dependent: :destroy + has_many :chat_names, foreign_key: :service_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent def valid_token?(token) self.respond_to?(:token) && diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index f38fbda7839..dfca0031af8 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -31,7 +31,7 @@ class ProjectWiki end def web_url - Gitlab::Routing.url_helpers.namespace_project_wiki_url(@project.namespace, @project, :home) + Gitlab::Routing.url_helpers.project_wiki_url(@project, :home) end def url_to_repo @@ -63,6 +63,10 @@ class ProjectWiki !!repository.exists? end + def has_home_page? + !!find_page('home') + end + # Returns an Array of Gitlab WikiPage instances or an # empty Array if this Wiki has no pages. def pages @@ -149,6 +153,10 @@ class ProjectWiki wiki end + def ensure_repository + create_repo! unless repository_exists? + end + def hook_attrs { web_url: web_url, diff --git a/app/models/repository.rb b/app/models/repository.rb index c67475357d9..50b7a477904 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -123,6 +123,7 @@ class Repository commits end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/384 def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0) unless exists? && has_visible_content? && query.present? return [] @@ -456,10 +457,6 @@ class Repository nil end - def blob_by_oid(oid) - Gitlab::Git::Blob.raw(self, oid) - end - def root_ref if raw_repository raw_repository.root_ref @@ -470,8 +467,17 @@ class Repository end cache_method :root_ref + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/314 def exists? - refs_directory_exists? + return false unless path_with_namespace + + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + if enabled + raw_repository.exists? + else + refs_directory_exists? + end + end end cache_method :exists? @@ -605,27 +611,12 @@ class Repository end end - # Returns url for submodule - # - # Ex. - # @repository.submodule_url_for('master', 'rack') - # # => git@localhost:rack.git - # - def submodule_url_for(ref, path) - if submodules(ref).any? - submodule = submodules(ref)[path] - - if submodule - submodule['url'] - end - end - end - def last_commit_for_path(sha, path) sha = last_commit_id_for_path(sha, path) commit(sha) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/383 def last_commit_id_for_path(sha, path) key = path.blank? ? "last_commit_id_for_path:#{sha}" : "last_commit_id_for_path:#{sha}:#{Digest::SHA1.hexdigest(path)}" @@ -947,7 +938,7 @@ class Repository def is_ancestor?(ancestor_id, descendant_id) return false if ancestor_id.nil? || descendant_id.nil? - + Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| if is_enabled raw_repository.is_ancestor?(ancestor_id, descendant_id) @@ -1094,8 +1085,8 @@ class Repository blob_data_at(sha, '.gitlab/route-map.yml') end - def gitlab_ci_yml_for(sha) - blob_data_at(sha, '.gitlab-ci.yml') + def gitlab_ci_yml_for(sha, path = '.gitlab-ci.yml') + blob_data_at(sha, path) end private @@ -1109,8 +1100,6 @@ class Repository end def refs_directory_exists? - return false unless path_with_namespace - File.exist?(File.join(path_to_repo, 'refs')) end diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index edde7bedbab..298569cb7a6 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -1,5 +1,5 @@ class SentNotification < ActiveRecord::Base - serialize :position, Gitlab::Diff::Position # rubocop:disable Cop/ActiverecordSerialize + serialize :position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize belongs_to :project belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations diff --git a/app/models/service.rb b/app/models/service.rb index 6a0b0a5c522..6b64079215f 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -2,7 +2,7 @@ # and implement a set of methods class Service < ActiveRecord::Base include Sortable - serialize :properties, JSON # rubocop:disable Cop/ActiverecordSerialize + serialize :properties, JSON # rubocop:disable Cop/ActiveRecordSerialize default_value_for :active, false default_value_for :push_events, true @@ -51,6 +51,14 @@ class Service < ActiveRecord::Base active end + def show_active_box? + true + end + + def editable? + true + end + def template? template end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 54014df43b0..09d5ff46618 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -30,16 +30,14 @@ class Snippet < ActiveRecord::Base belongs_to :author, class_name: 'User' belongs_to :project - has_many :notes, as: :noteable, dependent: :destroy + has_many :notes, as: :noteable, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent delegate :name, :email, to: :author, prefix: true, allow_nil: true validates :author, presence: true validates :title, presence: true, length: { maximum: 255 } validates :file_name, - length: { maximum: 255 }, - format: { with: Gitlab::Regex.file_name_regex, - message: Gitlab::Regex.file_name_regex_message } + length: { maximum: 255 } validates :content, presence: true validates :visibility_level, inclusion: { in: Gitlab::VisibilityLevel.values } diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 414c95f7705..0b33e45473b 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -1,7 +1,8 @@ class SystemNoteMetadata < ActiveRecord::Base ICON_TYPES = %w[ commit description merge confidential visible label assignee cross_reference - title time_tracking branch milestone discussion task moved opened closed merged + title time_tracking branch milestone discussion task moved + opened closed merged duplicate outdated ].freeze diff --git a/app/models/user.rb b/app/models/user.rb index 954a30155f7..6e66c587a1f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -11,6 +11,8 @@ class User < ActiveRecord::Base include CaseSensitivity include TokenAuthenticatable include IgnorableColumn + include FeatureGate + include CreatedAtFilterable DEFAULT_NOTIFICATION_LEVEL = :participating @@ -40,7 +42,7 @@ class User < ActiveRecord::Base otp_secret_encryption_key: Gitlab::Application.secrets.otp_key_base devise :two_factor_backupable, otp_number_of_backup_codes: 10 - serialize :otp_backup_codes, JSON # rubocop:disable Cop/ActiverecordSerialize + serialize :otp_backup_codes, JSON # rubocop:disable Cop/ActiveRecordSerialize devise :lockable, :recoverable, :rememberable, :trackable, :validatable, :omniauthable, :confirmable, :registerable @@ -53,7 +55,7 @@ class User < ActiveRecord::Base lease = Gitlab::ExclusiveLease.new("user_update_tracked_fields:#{id}", timeout: 1.hour.to_i) return unless lease.try_obtain - save(validate: false) + Users::UpdateService.new(self).execute(validate: false) end attr_accessor :force_random_password @@ -66,24 +68,25 @@ class User < ActiveRecord::Base # # Namespace for personal projects - has_one :namespace, -> { where type: nil }, dependent: :destroy, foreign_key: :owner_id, autosave: true + has_one :namespace, -> { where type: nil }, dependent: :destroy, foreign_key: :owner_id, autosave: true # rubocop:disable Cop/ActiveRecordDependent # Profile has_many :keys, -> do type = Key.arel_table[:type] where(type.not_eq('DeployKey').or(type.eq(nil))) - end, dependent: :destroy - has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :destroy + end, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :gpg_keys - has_many :emails, dependent: :destroy - has_many :personal_access_tokens, dependent: :destroy - has_many :identities, dependent: :destroy, autosave: true - has_many :u2f_registrations, dependent: :destroy - has_many :chat_names, dependent: :destroy + has_many :emails, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :personal_access_tokens, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :identities, dependent: :destroy, autosave: true # rubocop:disable Cop/ActiveRecordDependent + has_many :u2f_registrations, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :chat_names, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Groups - has_many :members, dependent: :destroy - has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, source: 'GroupMember' + has_many :members, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, source: 'GroupMember' # rubocop:disable Cop/ActiveRecordDependent has_many :groups, through: :group_members has_many :owned_groups, -> { where members: { access_level: Gitlab::Access::OWNER } }, through: :group_members, source: :group has_many :masters_groups, -> { where members: { access_level: Gitlab::Access::MASTER } }, through: :group_members, source: :group @@ -91,35 +94,35 @@ class User < ActiveRecord::Base # Projects has_many :groups_projects, through: :groups, source: :projects has_many :personal_projects, through: :namespace, source: :projects - has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy + has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :project_members has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' - has_many :users_star_projects, dependent: :destroy + has_many :users_star_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :starred_projects, through: :users_star_projects, source: :project has_many :project_authorizations has_many :authorized_projects, through: :project_authorizations, source: :project - has_many :snippets, dependent: :destroy, foreign_key: :author_id - has_many :notes, dependent: :destroy, foreign_key: :author_id - has_many :issues, dependent: :destroy, foreign_key: :author_id - has_many :merge_requests, dependent: :destroy, foreign_key: :author_id - has_many :events, dependent: :destroy, foreign_key: :author_id - has_many :subscriptions, dependent: :destroy + has_many :snippets, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :notes, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :issues, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :merge_requests, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :events, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :subscriptions, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :recent_events, -> { order "id DESC" }, foreign_key: :author_id, class_name: "Event" - has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy - has_one :abuse_report, dependent: :destroy, foreign_key: :user_id - has_many :reported_abuse_reports, dependent: :destroy, foreign_key: :reporter_id, class_name: "AbuseReport" - has_many :spam_logs, dependent: :destroy - has_many :builds, dependent: :nullify, class_name: 'Ci::Build' - has_many :pipelines, dependent: :nullify, class_name: 'Ci::Pipeline' - has_many :todos, dependent: :destroy - has_many :notification_settings, dependent: :destroy - has_many :award_emoji, dependent: :destroy - has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id + has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :abuse_report, dependent: :destroy, foreign_key: :user_id # rubocop:disable Cop/ActiveRecordDependent + has_many :reported_abuse_reports, dependent: :destroy, foreign_key: :reporter_id, class_name: "AbuseReport" # rubocop:disable Cop/ActiveRecordDependent + has_many :spam_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :builds, dependent: :nullify, class_name: 'Ci::Build' # rubocop:disable Cop/ActiveRecordDependent + has_many :pipelines, dependent: :nullify, class_name: 'Ci::Pipeline' # rubocop:disable Cop/ActiveRecordDependent + has_many :todos, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :notification_settings, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id # rubocop:disable Cop/ActiveRecordDependent has_many :issue_assignees has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue - has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" + has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent # # Validations @@ -155,6 +158,7 @@ class User < ActiveRecord::Base before_save :ensure_authentication_token, :ensure_incoming_email_token before_save :ensure_user_rights_and_limits, if: :external_changed? after_save :ensure_namespace_correct + after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') } after_initialize :set_projects_limit after_destroy :post_destroy_hook @@ -210,7 +214,7 @@ class User < ActiveRecord::Base end mount_uploader :avatar, AvatarUploader - has_many :uploads, as: :model, dependent: :destroy + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Scopes scope :admins, -> { where(admin: true) } @@ -299,11 +303,20 @@ class User < ActiveRecord::Base table = arel_table pattern = "%#{query}%" + order = <<~SQL + CASE + WHEN users.name = %{query} THEN 0 + WHEN users.username = %{query} THEN 1 + WHEN users.email = %{query} THEN 2 + ELSE 3 + END + SQL + where( table[:name].matches(pattern) .or(table[:email].matches(pattern)) .or(table[:username].matches(pattern)) - ) + ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) end # searches user by given pattern @@ -374,9 +387,11 @@ class User < ActiveRecord::Base # Return (create if necessary) the ghost user. The ghost user # owns records previously belonging to deleted users. def ghost - unique_internal(where(ghost: true), 'ghost', 'ghost%s@example.com') do |u| + email = 'ghost%s@example.com' + unique_internal(where(ghost: true), 'ghost', email) do |u| u.bio = 'This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.' u.name = 'Ghost User' + u.notification_email = email end end end @@ -494,13 +509,15 @@ class User < ActiveRecord::Base def update_emails_with_primary_email primary_email_record = emails.find_by(email: email) if primary_email_record - primary_email_record.destroy - emails.create(email: email_was) - - update_secondary_emails! + Emails::DestroyService.new(self, email: email).execute + Emails::CreateService.new(self, email: email_was).execute end end + def update_invalid_gpg_signatures + gpg_keys.each(&:update_invalid_gpg_signatures) + end + # Returns the groups a user has access to def authorized_groups union = Gitlab::SQL::Union @@ -571,8 +588,18 @@ class User < ActiveRecord::Base keys.count == 0 && Gitlab::ProtocolAccess.allowed?('ssh') end - def require_password? - password_automatically_set? && !ldap_user? + def require_password_creation? + password_automatically_set? && allow_password_authentication? + end + + def require_personal_access_token_creation_for_git_auth? + return false if allow_password_authentication? || ldap_user? + + PersonalAccessTokensFinder.new(user: self, impersonation: false, state: 'active').execute.none? + end + + def allow_password_authentication? + !ldap_user? && current_application_settings.password_authentication_enabled? end def can_change_username? @@ -684,7 +711,7 @@ class User < ActiveRecord::Base end def sanitize_attrs - %w[name username skype linkedin twitter].each do |attr| + %w[username skype linkedin twitter].each do |attr| value = public_send(attr) public_send("#{attr}=", Sanitize.clean(value)) if value.present? end @@ -965,7 +992,7 @@ class User < ActiveRecord::Base if attempts_exceeded? lock_access! unless access_locked? else - save(validate: false) + Users::UpdateService.new(self).execute(validate: false) end end @@ -984,6 +1011,12 @@ class User < ActiveRecord::Base self.admin = (new_level == 'admin') end + # Does the user have access to all private groups & projects? + # Overridden in EE to also check auditor? + def full_private_access? + admin? + end + def update_two_factor_requirement periods = expanded_groups_requiring_two_factor_authentication.pluck(:two_factor_grace_period) @@ -1123,7 +1156,8 @@ class User < ActiveRecord::Base email: email, &creation_block ) - user.save(validate: false) + + Users::UpdateService.new(user).execute(validate: false) user ensure Gitlab::ExclusiveLease.cancel(lease_key, uuid) diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 224eb3cd4d0..148998bc9be 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -1,4 +1,6 @@ class WikiPage + PageChangedError = Class.new(StandardError) + include ActiveModel::Validations include ActiveModel::Conversion include StaticModel @@ -136,6 +138,10 @@ class WikiPage versions.first end + def last_commit_sha + commit&.sha + end + # Returns the Date that this latest version was # created on. def created_at @@ -182,17 +188,22 @@ class WikiPage # Updates an existing Wiki Page, creating a new version. # - # new_content - The raw markup content to replace the existing. - # format - Optional symbol representing the content format. - # See ProjectWiki::MARKUPS Hash for available formats. - # message - Optional commit message to set on the new version. + # new_content - The raw markup content to replace the existing. + # format - Optional symbol representing the content format. + # See ProjectWiki::MARKUPS Hash for available formats. + # message - Optional commit message to set on the new version. + # last_commit_sha - Optional last commit sha to validate the page unchanged. # # Returns the String SHA1 of the newly created page # or False if the save was unsuccessful. - def update(new_content = "", format = :markdown, message = nil) + def update(new_content, format: :markdown, message: nil, last_commit_sha: nil) @attributes[:content] = new_content @attributes[:format] = format + if last_commit_sha && last_commit_sha != self.last_commit_sha + raise PageChangedError.new("You are attempting to update a page that has changed since you started editing it.") + end + save :update_page, @page, content, format, message end |