diff options
Diffstat (limited to 'app/models')
70 files changed, 1275 insertions, 354 deletions
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index c0cc60d5ebf..5e16badabec 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -33,6 +33,8 @@ class ApplicationSetting < ActiveRecord::Base attr_accessor :domain_whitelist_raw, :domain_blacklist_raw + default_value_for :id, 1 + validates :uuid, presence: true validates :session_expire_delay, @@ -151,6 +153,25 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { greater_than_or_equal_to: 0 } + validates :circuitbreaker_backoff_threshold, + :circuitbreaker_failure_count_threshold, + :circuitbreaker_failure_wait_time, + :circuitbreaker_failure_reset_time, + :circuitbreaker_storage_timeout, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + validates :circuitbreaker_access_retries, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 1 } + + validates_each :circuitbreaker_backoff_threshold do |record, attr, value| + if value.to_i >= record.circuitbreaker_failure_count_threshold + record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\ + "lower than the failure count threshold")) + end + end + SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end @@ -194,7 +215,10 @@ class ApplicationSetting < ActiveRecord::Base ensure_cache_setup Rails.cache.fetch(CACHE_KEY) do - ApplicationSetting.last + ApplicationSetting.last.tap do |settings| + # do not cache nils + raise 'missing settings' unless settings + end end rescue # Fall back to an uncached value if there are any problems (e.g. redis down) @@ -396,7 +420,7 @@ class ApplicationSetting < ActiveRecord::Base # the enabling/disabling is `performance_bar_allowed_group_id` # - If `enable` is false, we set `performance_bar_allowed_group_id` to `nil` def performance_bar_enabled=(enable) - return if enable + return if Gitlab::Utils.to_boolean(enable) self.performance_bar_allowed_group_id = nil end diff --git a/app/models/blob.rb b/app/models/blob.rb index 954d4e4d779..ad0bc2e2ead 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -156,7 +156,9 @@ class Blob < SimpleDelegator end def file_type - Gitlab::FileDetector.type_of(path) + name = File.basename(path) + + Gitlab::FileDetector.type_of(path) || Gitlab::FileDetector.type_of(name) end def video? diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb index b35febc9ac5..ec56cc53aea 100644 --- a/app/models/ci/artifact_blob.rb +++ b/app/models/ci/artifact_blob.rb @@ -2,6 +2,8 @@ module Ci class ArtifactBlob include BlobLike + EXTENSIONS_SERVED_BY_PAGES = %w[.html .htm .txt .json].freeze + attr_reader :entry def initialize(entry) @@ -17,6 +19,7 @@ module Ci def size entry.metadata[:size] end + alias_method :external_size, :size def data "Build artifact #{path}" @@ -30,6 +33,32 @@ module Ci :build_artifact end - alias_method :external_size, :size + def external_url(project, job) + return unless external_link?(job) + + full_path_parts = project.full_path_components + top_level_group = full_path_parts.shift + + artifact_path = [ + '-', *full_path_parts, '-', + 'jobs', job.id, + 'artifacts', path + ].join('/') + + "#{pages_config.protocol}://#{top_level_group}.#{pages_config.host}/#{artifact_path}" + end + + def external_link?(job) + pages_config.enabled && + pages_config.artifacts_server && + EXTENSIONS_SERVED_BY_PAGES.include?(File.extname(name)) && + job.project.public? + end + + private + + def pages_config + Gitlab.config.pages + end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index ee544d8ac56..6ca46ae89c1 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -11,6 +11,7 @@ module Ci has_many :deployments, as: :deployable has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' + has_many :trace_sections, class_name: 'Ci::BuildTraceSection' # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @@ -229,6 +230,10 @@ module Ci variables end + def features + { trace_sections: true } + end + def merge_request return @merge_request if defined?(@merge_request) @@ -261,6 +266,10 @@ module Ci update_attributes(coverage: coverage) if coverage.present? end + def parse_trace_sections! + ExtractSectionsFromBuildTraceService.new(project, user).execute(self) + end + def trace Gitlab::Ci::Trace.new(self) end diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb new file mode 100644 index 00000000000..ccdb95546c8 --- /dev/null +++ b/app/models/ci/build_trace_section.rb @@ -0,0 +1,11 @@ +module Ci + class BuildTraceSection < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :build, class_name: 'Ci::Build' + belongs_to :project + belongs_to :section_name, class_name: 'Ci::BuildTraceSectionName' + + validates :section_name, :build, :project, presence: true, allow_blank: false + end +end diff --git a/app/models/ci/build_trace_section_name.rb b/app/models/ci/build_trace_section_name.rb new file mode 100644 index 00000000000..0fdcb1ea329 --- /dev/null +++ b/app/models/ci/build_trace_section_name.rb @@ -0,0 +1,11 @@ +module Ci + class BuildTraceSectionName < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :project + has_many :trace_sections, class_name: 'Ci::BuildTraceSection', foreign_key: :section_name_id + + validates :name, :project, presence: true, allow_blank: false + validates :name, uniqueness: { scope: :project_id } + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index acaa028eaa2..ca65e81f27a 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -5,6 +5,7 @@ module Ci include Importable include AfterCommitQueue include Presentable + include Gitlab::OptimisticLocking belongs_to :project belongs_to :user @@ -58,6 +59,11 @@ module Ci auto_devops_source: 2 } + enum failure_reason: { + unknown_failure: 0, + config_error: 1 + } + state_machine :status, initial: :created do event :enqueue do transition created: :pending @@ -109,6 +115,12 @@ module Ci pipeline.auto_canceled_by = nil end + before_transition any => :failed do |pipeline, transition| + transition.args.first.try do |reason| + pipeline.failure_reason = reason + end + end + after_transition [:created, :pending] => :running do |pipeline| pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) } end @@ -237,9 +249,7 @@ module Ci end def commit - @commit ||= project.commit(sha) - rescue - nil + @commit ||= project.commit_by(oid: sha) end def branch? @@ -263,7 +273,7 @@ module Ci end def cancel_running - Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable| + retry_optimistic_lock(cancelable_statuses) do |cancelable| cancelable.find_each do |job| yield(job) if block_given? job.cancel @@ -312,6 +322,10 @@ module Ci @stage_seeds ||= config_processor.stage_seeds(self) end + def seeds_size + @seeds_size ||= stage_seeds.sum(&:size) + end + def has_kubernetes_active? project.kubernetes_service&.active? end @@ -403,7 +417,7 @@ module Ci end def update_status - Gitlab::OptimisticLocking.retry_lock(self) do + retry_optimistic_lock(self) do case latest_builds_status when 'pending' then enqueue when 'running' then run @@ -434,7 +448,7 @@ module Ci def update_duration return unless started_at - self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self) + self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self) end def execute_hooks diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index a0d07902ba2..c6509f89117 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -174,7 +174,7 @@ module Ci end def assignable_for?(project) - !locked? || projects.exists?(id: project.id) + is_shared? || projects.exists?(id: project.id) end def accepting_tags?(build) diff --git a/app/models/commit.rb b/app/models/commit.rb index 2ae8890c1b3..6dba154a6ea 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -25,8 +25,8 @@ class Commit DIFF_HARD_LIMIT_FILES = 1000 DIFF_HARD_LIMIT_LINES = 50000 - # The SHA can be between 7 and 40 hex characters. - COMMIT_SHA_PATTERN = '\h{7,40}'.freeze + MIN_SHA_LENGTH = 7 + COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze def banzai_render_context(field) context = { pipeline: :single_line, project: self.project } @@ -53,7 +53,7 @@ class Commit # Truncate sha to 8 characters def truncate_sha(sha) - sha[0..7] + sha[0..MIN_SHA_LENGTH] end def max_diff_options @@ -100,7 +100,7 @@ class Commit def self.reference_pattern @reference_pattern ||= %r{ (?:#{Project.reference_pattern}#{reference_prefix})? - (?<commit>\h{7,40}) + (?<commit>#{COMMIT_SHA_PATTERN}) }x end @@ -216,9 +216,8 @@ class Commit @raw.respond_to?(method, include_private) || super end - # Truncate sha to 8 characters def short_id - @raw.short_id(7) + @raw.short_id(MIN_SHA_LENGTH) end def diff_refs diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 8fbfed11bdf..2ec70203710 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -11,7 +11,7 @@ module Avatarable # If asset_host is set then it is expected that assets are handled by a standalone host. # That means we do not want to get GitLab's relative_url_root option anymore. - host = asset_host.present? ? asset_host : gitlab_host + host = (asset_host.present? && (!respond_to?(:public?) || public?)) ? asset_host : gitlab_host [host, avatar.url].join end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 193e459977a..98776eab424 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -49,7 +49,8 @@ module CacheMarkdownField # Always include a project key, or Banzai complains project = self.project if self.respond_to?(:project) - context = cached_markdown_fields[field].merge(project: project) + group = self.group if self.respond_to?(:group) + context = cached_markdown_fields[field].merge(project: project, group: group) # Banzai is less strict about authors, so don't always have an author key context[:author] = self.author if self.respond_to?(:author) @@ -59,7 +60,7 @@ module CacheMarkdownField # Update every column in a row if any one is invalidated, as we only store # one version per row - def refresh_markdown_cache!(do_update: false) + def refresh_markdown_cache options = { skip_project_check: skip_project_check? } updates = cached_markdown_fields.markdown_fields.map do |markdown_field| @@ -71,8 +72,14 @@ module CacheMarkdownField updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION updates.each {|html_field, data| write_attribute(html_field, data) } + end + + def refresh_markdown_cache! + updates = refresh_markdown_cache + + return unless persisted? && Gitlab::Database.read_write? - update_columns(updates) if persisted? && do_update + update_columns(updates) end def cached_html_up_to_date?(markdown_field) @@ -124,8 +131,8 @@ module CacheMarkdownField end # Using before_update here conflicts with elasticsearch-model somehow - before_create :refresh_markdown_cache!, if: :invalidated_markdown_cache? - before_update :refresh_markdown_cache!, if: :invalidated_markdown_cache? + before_create :refresh_markdown_cache, if: :invalidated_markdown_cache? + before_update :refresh_markdown_cache, if: :invalidated_markdown_cache? end class_methods do diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb index eee1a36ac6b..f5cbb3becad 100644 --- a/app/models/concerns/discussion_on_diff.rb +++ b/app/models/concerns/discussion_on_diff.rb @@ -28,6 +28,10 @@ module DiscussionOnDiff true end + def file_new_path + first_note.position.new_path + end + # Returns an array of at most 16 highlighted lines above a diff note def truncated_diff_lines(highlight: true) lines = highlight ? highlighted_diff_lines : diff_lines diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb new file mode 100644 index 00000000000..01957da0bf3 --- /dev/null +++ b/app/models/concerns/group_descendant.rb @@ -0,0 +1,56 @@ +module GroupDescendant + # Returns the hierarchy of a project or group in the from of a hash upto a + # given top. + # + # > project.hierarchy + # => { parent_group => { child_group => project } } + def hierarchy(hierarchy_top = nil, preloaded = nil) + preloaded ||= ancestors_upto(hierarchy_top) + expand_hierarchy_for_child(self, self, hierarchy_top, preloaded) + end + + # Merges all hierarchies of the given groups or projects into an array of + # hashes. All ancestors need to be loaded into the given `descendants` to avoid + # queries down the line. + # + # > GroupDescendant.merge_hierarchy([project, child_group, child_group2, parent]) + # => { parent => [{ child_group => project}, child_group2] } + def self.build_hierarchy(descendants, hierarchy_top = nil) + descendants = Array.wrap(descendants).uniq + return [] if descendants.empty? + + unless descendants.all? { |hierarchy| hierarchy.is_a?(GroupDescendant) } + raise ArgumentError.new('element is not a hierarchy') + end + + all_hierarchies = descendants.map do |descendant| + descendant.hierarchy(hierarchy_top, descendants) + end + + Gitlab::Utils::MergeHash.merge(all_hierarchies) + end + + private + + def expand_hierarchy_for_child(child, hierarchy, hierarchy_top, preloaded) + parent = hierarchy_top if hierarchy_top && child.parent_id == hierarchy_top.id + parent ||= preloaded.detect { |possible_parent| possible_parent.is_a?(Group) && possible_parent.id == child.parent_id } + + if parent.nil? && !child.parent_id.nil? + raise ArgumentError.new('parent was not preloaded') + end + + if parent.nil? && hierarchy_top.present? + raise ArgumentError.new('specified top is not part of the tree') + end + + if parent && parent != hierarchy_top + expand_hierarchy_for_child(parent, + { parent => hierarchy }, + hierarchy_top, + preloaded) + else + hierarchy + end + end +end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 3803e18a96e..7c3ed96bc28 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -81,6 +81,7 @@ module HasStatus scope :canceled, -> { where(status: 'canceled') } scope :skipped, -> { where(status: 'skipped') } scope :manual, -> { where(status: 'manual') } + scope :alive, -> { where(status: [:created, :pending, :running]) } scope :created_or_pending, -> { where(status: [:created, :pending]) } scope :running_or_pending, -> { where(status: [:running, :pending]) } scope :finished, -> { where(status: [:success, :failed, :canceled]) } diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 265f6e48540..a928b9d6367 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -14,7 +14,6 @@ module Issuable include StripAttribute include Awardable include Taskable - include TimeTrackable include Importable include Editable include AfterCommitQueue @@ -95,8 +94,6 @@ module Issuable strip_attributes :title - acts_as_paranoid - after_save :record_metrics, unless: :imported? # We want to use optimistic lock for cases when only title or description are involved @@ -143,16 +140,18 @@ module Issuable end def sort(method, excluded_labels: []) - sorted = case method.to_s - when 'milestone_due_asc' then order_milestone_due_asc - when 'milestone_due_desc' then order_milestone_due_desc - when 'downvotes_desc' then order_downvotes_desc - when 'upvotes_desc' then order_upvotes_desc - when 'label_priority' then order_labels_priority(excluded_labels: excluded_labels) - when 'priority' then order_due_date_and_labels_priority(excluded_labels: excluded_labels) - else - order_by(method) - end + sorted = + case method.to_s + when 'downvotes_desc' then order_downvotes_desc + when 'label_priority' then order_labels_priority(excluded_labels: excluded_labels) + when 'milestone' then order_milestone_due_asc + when 'milestone_due_asc' then order_milestone_due_asc + when 'milestone_due_desc' then order_milestone_due_desc + when 'popularity' then order_upvotes_desc + when 'priority' then order_due_date_and_labels_priority(excluded_labels: excluded_labels) + when 'upvotes_desc' then order_upvotes_desc + else order_by(method) + end # Break ties with the ID column for pagination sorted.order(id: :desc) @@ -214,7 +213,7 @@ module Issuable def grouping_columns(sort) grouping_columns = [arel_table[:id]] - if %w(milestone_due_desc milestone_due_asc).include?(sort) + if %w(milestone_due_desc milestone_due_asc milestone).include?(sort) milestone_table = Milestone.arel_table grouping_columns << milestone_table[:id] grouping_columns << milestone_table[:due_date] @@ -254,23 +253,22 @@ module Issuable participants(user).include?(user) end - def to_hook_data(user) - hook_data = { - object_kind: self.class.name.underscore, - user: user.hook_attrs, - project: project.hook_attrs, - object_attributes: hook_attrs, - labels: labels.map(&:hook_attrs), - # DEPRECATED - repository: project.hook_attrs.slice(:name, :url, :description, :homepage) - } - if self.is_a?(Issue) - hook_data[:assignees] = assignees.map(&:hook_attrs) if assignees.any? - else - hook_data[:assignee] = assignee.hook_attrs if assignee + def to_hook_data(user, old_labels: [], old_assignees: []) + changes = previous_changes + + if old_labels != labels + changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)] + end + + if old_assignees != assignees + if self.is_a?(Issue) + changes[:assignees] = [old_assignees.map(&:hook_attrs), assignees.map(&:hook_attrs)] + else + changes[:assignee] = [old_assignees&.first&.hook_attrs, assignee&.hook_attrs] + end end - hook_data + Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes) end def labels_array diff --git a/app/models/concerns/loaded_in_group_list.rb b/app/models/concerns/loaded_in_group_list.rb new file mode 100644 index 00000000000..dcb3b2b5ff3 --- /dev/null +++ b/app/models/concerns/loaded_in_group_list.rb @@ -0,0 +1,72 @@ +module LoadedInGroupList + extend ActiveSupport::Concern + + module ClassMethods + def with_counts(archived:) + selects_including_counts = [ + 'namespaces.*', + "(#{project_count_sql(archived).to_sql}) AS preloaded_project_count", + "(#{member_count_sql.to_sql}) AS preloaded_member_count", + "(#{subgroup_count_sql.to_sql}) AS preloaded_subgroup_count" + ] + + select(selects_including_counts) + end + + def with_selects_for_list(archived: nil) + with_route.with_counts(archived: archived) + end + + private + + def project_count_sql(archived = nil) + projects = Project.arel_table + namespaces = Namespace.arel_table + + base_count = projects.project(Arel.star.count.as('preloaded_project_count')) + .where(projects[:namespace_id].eq(namespaces[:id])) + if archived == 'only' + base_count.where(projects[:archived].eq(true)) + elsif Gitlab::Utils.to_boolean(archived) + base_count + else + base_count.where(projects[:archived].not_eq(true)) + end + end + + def subgroup_count_sql + namespaces = Namespace.arel_table + children = namespaces.alias('children') + + namespaces.project(Arel.star.count.as('preloaded_subgroup_count')) + .from(children) + .where(children[:parent_id].eq(namespaces[:id])) + end + + def member_count_sql + members = Member.arel_table + namespaces = Namespace.arel_table + + members.project(Arel.star.count.as('preloaded_member_count')) + .where(members[:source_type].eq(Namespace.name)) + .where(members[:source_id].eq(namespaces[:id])) + .where(members[:requested_at].eq(nil)) + end + end + + def children_count + @children_count ||= project_count + subgroup_count + end + + def project_count + @project_count ||= try(:preloaded_project_count) || projects.non_archived.count + end + + def subgroup_count + @subgroup_count ||= try(:preloaded_subgroup_count) || children.count + end + + def member_count + @member_count ||= try(:preloaded_member_count) || users.count + end +end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index 9d81a19cbb9..b44274f6145 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -75,4 +75,8 @@ module Noteable def discussions_can_be_resolved_by?(user) discussions_to_be_resolved.all? { |discussion| discussion.can_resolve?(user) } end + + def lockable? + [MergeRequest, Issue].include?(self.class) + end end diff --git a/app/models/concerns/repository_mirroring.rb b/app/models/concerns/repository_mirroring.rb deleted file mode 100644 index fed336c29d6..00000000000 --- a/app/models/concerns/repository_mirroring.rb +++ /dev/null @@ -1,17 +0,0 @@ -module RepositoryMirroring - def set_remote_as_mirror(name) - config = raw_repository.rugged.config - - # This is used to define repository as equivalent as "git clone --mirror" - config["remote.#{name}.fetch"] = 'refs/*:refs/*' - config["remote.#{name}.mirror"] = true - config["remote.#{name}.prune"] = true - end - - def fetch_mirror(remote, url) - add_remote(remote, url) - set_remote_as_mirror(remote) - fetch_remote(remote, forced: true) - remove_remote(remote) - end -end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 80a8f63514f..05ddae42d2d 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -107,7 +107,10 @@ module Routable RequestStore[full_path_key] ||= uncached_full_path end - # rubocop:disable Cop/ModuleWithInstanceVariables + def full_path_components + full_path.split('/') + end + def expires_full_path_cache RequestStore.delete(full_path_key) if RequestStore.active? @full_path = nil @@ -155,6 +158,8 @@ module Routable end def update_route + return if Gitlab::Database.read_only? + prepare_route route.save end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index db3cd257584..cefa5c13c5f 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -19,14 +19,15 @@ module Sortable module ClassMethods def order_by(method) case method.to_s - when 'name_asc' then order_name_asc - when 'name_desc' then order_name_desc - when 'updated_asc' then order_updated_asc - when 'updated_desc' then order_updated_desc - when 'created_asc' then order_created_asc + when 'created_asc' then order_created_asc + when 'created_date' then order_created_desc when 'created_desc' then order_created_desc - when 'id_desc' then order_id_desc - when 'id_asc' then order_id_asc + when 'id_asc' then order_id_asc + when 'id_desc' then order_id_desc + when 'name_asc' then order_name_asc + when 'name_desc' then order_name_desc + when 'updated_asc' then order_updated_asc + when 'updated_desc' then order_updated_desc else all end diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index 5ab5c80a2f5..b3020484738 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -7,6 +7,8 @@ module Storage raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') end + expires_full_path_cache + # Move the namespace directory in all storage paths used by member projects repository_storage_paths.each do |repository_storage_path| # Ensure old directory exists before moving it diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index 274b38a7708..f478c8ede18 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -13,6 +13,8 @@ module Subscribable end def subscribed?(user, project = nil) + return false unless user + if subscription = subscriptions.find_by(user: user, project: project) subscription.subscribed else diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 995fa98efac..49438908c36 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -8,7 +8,7 @@ module TimeTrackable extend ActiveSupport::Concern included do - attr_reader :time_spent, :time_spent_user + attr_reader :time_spent, :time_spent_user, :spent_at alias_method :time_spent?, :time_spent @@ -24,6 +24,7 @@ module TimeTrackable def spend_time(options) @time_spent = options[:duration] @time_spent_user = options[:user] + @spent_at = options[:spent_at] @original_total_time_spent = nil return if @time_spent == 0 @@ -56,7 +57,11 @@ module TimeTrackable end def add_or_subtract_spent_time - timelogs.new(time_spent: time_spent, user: @time_spent_user) + timelogs.new( + time_spent: time_spent, + user: @time_spent_user, + spent_at: @spent_at + ) end # rubocop:disable Cop/ModuleWithInstanceVariables diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index a7d5de48c66..ec3543f7053 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -43,15 +43,17 @@ module TokenAuthenticatable write_attribute(token_field, token) if token end + # Returns a token, but only saves when the database is in read & write mode define_method("ensure_#{token_field}!") do send("reset_#{token_field}!") if read_attribute(token_field).blank? # rubocop:disable GitlabSecurity/PublicSend read_attribute(token_field) end + # Resets the token, but only saves when the database is in read & write mode define_method("reset_#{token_field}!") do write_new_token(token_field) - save! + save! if Gitlab::Database.read_write? end end end diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb index 07c4846e2ac..6eba87da1a1 100644 --- a/app/models/diff_discussion.rb +++ b/app/models/diff_discussion.rb @@ -11,6 +11,8 @@ class DiffDiscussion < Discussion delegate :position, :original_position, :change_position, + :on_text?, + :on_image?, to: :first_note diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index e9a60e6ce09..d88a92dc027 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -12,8 +12,8 @@ class DiffNote < Note validates :original_position, presence: true validates :position, presence: true - validates :diff_line, presence: true - validates :line_code, presence: true, line_code: true + validates :diff_line, presence: true, if: :on_text? + validates :line_code, presence: true, line_code: true, if: :on_text? validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } validate :positions_complete validate :verify_supported @@ -43,6 +43,14 @@ class DiffNote < Note end end + def on_text? + position.position_type == "text" + end + + def on_image? + position.position_type == "image" + end + def diff_file @diff_file ||= self.original_position.diff_file(self.project.repository) end @@ -56,6 +64,8 @@ class DiffNote < Note end def original_line_code + return unless on_text? + self.diff_file.line_code(self.diff_line) end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index b80da7b246a..437df923d2d 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -66,6 +66,10 @@ class Discussion @context_noteable = context_noteable end + def on_image? + false + end + def ==(other) other.class == self.class && other.context_noteable == self.context_noteable && diff --git a/app/models/email.rb b/app/models/email.rb index 826d4f16edb..2da8b050149 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -7,6 +7,15 @@ class Email < ActiveRecord::Base validates :email, presence: true, uniqueness: true, email: true validate :unique_email, if: ->(email) { email.email_changed? } + scope :confirmed, -> { where.not(confirmed_at: nil) } + + after_commit :update_invalid_gpg_signatures, if: -> { previous_changes.key?('confirmed_at') } + + devise :confirmable + self.reconfirmable = false # currently email can't be changed, no need to reconfirm + + delegate :username, to: :user + def email=(value) write_attribute(:email, value.downcase.strip) end @@ -14,4 +23,9 @@ class Email < ActiveRecord::Base def unique_email self.errors.add(:email, 'has already been taken') if User.exists?(email: self.email) end + + # once email is confirmed, update the gpg signatures + def update_invalid_gpg_signatures + user.update_invalid_gpg_signatures if confirmed? + end end diff --git a/app/models/environment.rb b/app/models/environment.rb index b6868ccbe8f..21a028e351c 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -30,7 +30,6 @@ class Environment < ActiveRecord::Base message: Gitlab::Regex.environment_slug_regex_message } validates :external_url, - uniqueness: { scope: :project_id }, length: { maximum: 255 }, allow_nil: true, addressable_url: true @@ -110,7 +109,7 @@ class Environment < ActiveRecord::Base end def ref_path - "refs/#{Repository::REF_ENVIRONMENTS}/#{Shellwords.shellescape(name)}" + "refs/#{Repository::REF_ENVIRONMENTS}/#{slug}" end def formatted_external_url @@ -164,6 +163,10 @@ class Environment < ActiveRecord::Base end end + def slug + super.presence || generate_slug + end + # An environment name is not necessarily suitable for use in URLs, DNS # or other third-party contexts, so provide a slugified version. A slug has # the following properties: diff --git a/app/models/epic.rb b/app/models/epic.rb new file mode 100644 index 00000000000..62898a02e2d --- /dev/null +++ b/app/models/epic.rb @@ -0,0 +1,7 @@ +# Placeholder class for model that is implemented in EE +# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE +class Epic < ActiveRecord::Base + # TODO: this will be implemented as part of #3853 + def to_reference + end +end diff --git a/app/models/fork_network.rb b/app/models/fork_network.rb new file mode 100644 index 00000000000..7f1728e8c77 --- /dev/null +++ b/app/models/fork_network.rb @@ -0,0 +1,19 @@ +class ForkNetwork < ActiveRecord::Base + belongs_to :root_project, class_name: 'Project' + has_many :fork_network_members + has_many :projects, through: :fork_network_members + + after_create :add_root_as_member, if: :root_project + + def add_root_as_member + projects << root_project + end + + def find_forks_in(other_projects) + projects.where(id: other_projects) + end + + def merge_requests + MergeRequest.where(target_project: projects) + end +end diff --git a/app/models/fork_network_member.rb b/app/models/fork_network_member.rb new file mode 100644 index 00000000000..6a9b52a1ef8 --- /dev/null +++ b/app/models/fork_network_member.rb @@ -0,0 +1,7 @@ +class ForkNetworkMember < ActiveRecord::Base + belongs_to :fork_network + belongs_to :project + belongs_to :forked_from_project, class_name: 'Project' + + validates :fork_network, :project, presence: true +end diff --git a/app/models/gcp/cluster.rb b/app/models/gcp/cluster.rb new file mode 100644 index 00000000000..162a690c0e3 --- /dev/null +++ b/app/models/gcp/cluster.rb @@ -0,0 +1,116 @@ +module Gcp + class Cluster < ActiveRecord::Base + extend Gitlab::Gcp::Model + include Presentable + + belongs_to :project, inverse_of: :cluster + belongs_to :user + belongs_to :service + + scope :enabled, -> { where(enabled: true) } + scope :disabled, -> { where(enabled: false) } + + default_value_for :gcp_cluster_zone, 'us-central1-a' + default_value_for :gcp_cluster_size, 3 + default_value_for :gcp_machine_type, 'n1-standard-4' + + attr_encrypted :password, + mode: :per_attribute_iv, + key: Gitlab::Application.secrets.db_key_base, + algorithm: 'aes-256-cbc' + + attr_encrypted :kubernetes_token, + mode: :per_attribute_iv, + key: Gitlab::Application.secrets.db_key_base, + algorithm: 'aes-256-cbc' + + attr_encrypted :gcp_token, + mode: :per_attribute_iv, + key: Gitlab::Application.secrets.db_key_base, + algorithm: 'aes-256-cbc' + + validates :gcp_project_id, + length: 1..63, + format: { + with: Gitlab::Regex.kubernetes_namespace_regex, + message: Gitlab::Regex.kubernetes_namespace_regex_message + } + + validates :gcp_cluster_name, + length: 1..63, + format: { + with: Gitlab::Regex.kubernetes_namespace_regex, + message: Gitlab::Regex.kubernetes_namespace_regex_message + } + + validates :gcp_cluster_zone, presence: true + + validates :gcp_cluster_size, + presence: true, + numericality: { + only_integer: true, + greater_than: 0 + } + + validates :project_namespace, + allow_blank: true, + length: 1..63, + format: { + with: Gitlab::Regex.kubernetes_namespace_regex, + message: Gitlab::Regex.kubernetes_namespace_regex_message + } + + # if we do not do status transition we prevent change + validate :restrict_modification, on: :update, unless: :status_changed? + + state_machine :status, initial: :scheduled do + state :scheduled, value: 1 + state :creating, value: 2 + state :created, value: 3 + state :errored, value: 4 + + event :make_creating do + transition any - [:creating] => :creating + end + + event :make_created do + transition any - [:created] => :created + end + + event :make_errored do + transition any - [:errored] => :errored + end + + before_transition any => [:errored, :created] do |cluster| + cluster.gcp_token = nil + cluster.gcp_operation_id = nil + end + + before_transition any => [:errored] do |cluster, transition| + status_reason = transition.args.first + cluster.status_reason = status_reason if status_reason + end + end + + def project_namespace_placeholder + "#{project.path}-#{project.id}" + end + + def on_creation? + scheduled? || creating? + end + + def api_url + 'https://' + endpoint if endpoint + end + + def restrict_modification + if on_creation? + errors.add(:base, "cannot modify during creation") + return false + end + + true + end + end +end diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb index 44deae4234b..44eda741679 100644 --- a/app/models/gpg_key.rb +++ b/app/models/gpg_key.rb @@ -9,6 +9,9 @@ class GpgKey < ActiveRecord::Base belongs_to :user has_many :gpg_signatures + has_many :subkeys, class_name: 'GpgKeySubkey' + + scope :with_subkeys, -> { includes(:subkeys) } validates :user, presence: true @@ -36,10 +39,12 @@ class GpgKey < ActiveRecord::Base before_validation :extract_fingerprint, :extract_primary_keyid after_commit :update_invalid_gpg_signatures, on: :create + after_create :generate_subkeys def primary_keyid super&.upcase end + alias_method :keyid, :primary_keyid def fingerprint super&.upcase @@ -49,6 +54,10 @@ class GpgKey < ActiveRecord::Base super(value&.strip) end + def keyids + [keyid].concat(subkeys.map(&:keyid)) + end + def user_infos @user_infos ||= Gitlab::Gpg.user_infos_from_key(key) end @@ -73,7 +82,7 @@ class GpgKey < ActiveRecord::Base end def verified_and_belongs_to_email?(email) - emails_with_verified_status.fetch(email, false) + emails_with_verified_status.fetch(email.downcase, false) end def update_invalid_gpg_signatures @@ -82,10 +91,11 @@ class GpgKey < ActiveRecord::Base def revoke GpgSignature - .where(gpg_key: self) + .with_key_and_subkeys(self) .where.not(verification_status: GpgSignature.verification_statuses[:unknown_key]) .update_all( gpg_key_id: nil, + gpg_key_subkey_id: nil, verification_status: GpgSignature.verification_statuses[:unknown_key], updated_at: Time.zone.now ) @@ -106,4 +116,12 @@ class GpgKey < ActiveRecord::Base # only allows one key self.primary_keyid = Gitlab::Gpg.primary_keyids_from_key(key).first end + + def generate_subkeys + gpg_subkeys = Gitlab::Gpg.subkeys_from_key(key) + + gpg_subkeys[primary_keyid]&.each do |subkey_data| + subkeys.create!(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint]) + end + end end diff --git a/app/models/gpg_key_subkey.rb b/app/models/gpg_key_subkey.rb new file mode 100644 index 00000000000..b57922aba30 --- /dev/null +++ b/app/models/gpg_key_subkey.rb @@ -0,0 +1,22 @@ +class GpgKeySubkey < ActiveRecord::Base + include ShaAttribute + + sha_attribute :keyid + sha_attribute :fingerprint + + belongs_to :gpg_key + + validates :gpg_key_id, presence: true + validates :fingerprint, :keyid, presence: true, uniqueness: true + + delegate :key, :user, :user_infos, :verified?, :verified_user_infos, + :verified_and_belongs_to_email?, to: :gpg_key + + def keyid + super&.upcase + end + + def fingerprint + super&.upcase + end +end diff --git a/app/models/gpg_signature.rb b/app/models/gpg_signature.rb index 1f047a32c84..bf88d75246f 100644 --- a/app/models/gpg_signature.rb +++ b/app/models/gpg_signature.rb @@ -15,11 +15,42 @@ class GpgSignature < ActiveRecord::Base belongs_to :project belongs_to :gpg_key + belongs_to :gpg_key_subkey validates :commit_sha, presence: true validates :project_id, presence: true validates :gpg_key_primary_keyid, presence: true + def self.with_key_and_subkeys(gpg_key) + subkey_ids = gpg_key.subkeys.pluck(:id) + + where( + arel_table[:gpg_key_id].eq(gpg_key.id).or( + arel_table[:gpg_key_subkey_id].in(subkey_ids) + ) + ) + end + + def gpg_key=(model) + case model + when GpgKey + super + when GpgKeySubkey + self.gpg_key_subkey = model + when NilClass + super + self.gpg_key_subkey = nil + end + end + + def gpg_key + if gpg_key_id + super + elsif gpg_key_subkey_id + gpg_key_subkey + end + end + def gpg_key_primary_keyid super&.upcase end @@ -29,6 +60,8 @@ class GpgSignature < ActiveRecord::Base end def gpg_commit + return unless commit + Gitlab::Gpg::Commit.new(commit) end end diff --git a/app/models/group.rb b/app/models/group.rb index e746e4a12c9..c660de7fcb6 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -6,6 +6,8 @@ class Group < Namespace include Avatarable include Referable include SelectForProjectAuthorization + include LoadedInGroupList + include GroupDescendant has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members @@ -40,6 +42,7 @@ class Group < Namespace after_create :post_create_hook after_destroy :post_destroy_hook after_save :update_two_factor_requirement + after_update :path_changed_hook, if: :path_changed? class << self def supports_nested_groups? @@ -178,6 +181,12 @@ class Group < Namespace add_user(user, :owner, current_user: current_user) end + def member?(user, min_access_level = Gitlab::Access::GUEST) + return false unless user + + max_member_access_for_user(user) >= min_access_level + end + def has_owner?(user) return false unless user @@ -287,6 +296,12 @@ class Group < Namespace list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten end + def full_path_was + return path_was unless has_parent? + + "#{parent.full_path}/#{path_was}" + end + private def update_two_factor_requirement @@ -295,6 +310,10 @@ class Group < Namespace users.find_each(&:update_two_factor_requirement) end + def path_changed_hook + system_hook_service.execute_hooks_for(self, :rename) + end + def visibility_level_allowed_by_parent return if visibility_level_allowed_by_parent? diff --git a/app/models/identity.rb b/app/models/identity.rb index 920a25932b4..ac8094b610e 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -7,7 +7,10 @@ class Identity < ActiveRecord::Base validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider } validates :user_id, uniqueness: { scope: :provider } - scope :with_extern_uid, ->(provider, extern_uid) { where(extern_uid: extern_uid, provider: provider) } + scope :with_extern_uid, ->(provider, extern_uid) do + extern_uid = Gitlab::LDAP::Person.normalize_dn(extern_uid) if provider.starts_with?('ldap') + where(extern_uid: extern_uid, provider: provider) + end def ldap? provider.starts_with?('ldap') diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb new file mode 100644 index 00000000000..b30b707e5fe --- /dev/null +++ b/app/models/instance_configuration.rb @@ -0,0 +1,71 @@ +require 'resolv' + +class InstanceConfiguration + SSH_ALGORITHMS = %w(DSA ECDSA ED25519 RSA).freeze + SSH_ALGORITHMS_PATH = '/etc/ssh/'.freeze + CACHE_KEY = 'instance_configuration'.freeze + EXPIRATION_TIME = 24.hours + + def settings + @configuration ||= Rails.cache.fetch(CACHE_KEY, expires_in: EXPIRATION_TIME) do + { ssh_algorithms_hashes: ssh_algorithms_hashes, + host: host, + gitlab_pages: gitlab_pages, + gitlab_ci: gitlab_ci }.deep_symbolize_keys + end + end + + private + + def ssh_algorithms_hashes + SSH_ALGORITHMS.map { |algo| ssh_algorithm_hashes(algo) }.compact + end + + def host + Settings.gitlab.host + end + + def gitlab_pages + Settings.pages.to_h.merge(ip_address: resolv_dns(Settings.pages.host)) + end + + def resolv_dns(dns) + Resolv.getaddress(dns) + rescue Resolv::ResolvError + end + + def gitlab_ci + Settings.gitlab_ci + .to_h + .merge(artifacts_max_size: { value: Settings.artifacts.max_size&.megabytes, + default: 100.megabytes }) + end + + def ssh_algorithm_file(algorithm) + File.join(SSH_ALGORITHMS_PATH, "ssh_host_#{algorithm.downcase}_key.pub") + end + + def ssh_algorithm_hashes(algorithm) + content = ssh_algorithm_file_content(algorithm) + return unless content.present? + + { name: algorithm, + md5: ssh_algorithm_md5(content), + sha256: ssh_algorithm_sha256(content) } + end + + def ssh_algorithm_file_content(algorithm) + file = ssh_algorithm_file(algorithm) + return unless File.exist?(file) + + File.read(file) + end + + def ssh_algorithm_md5(ssh_file_content) + OpenSSL::Digest::MD5.hexdigest(ssh_file_content).scan(/../).join(':') + end + + def ssh_algorithm_sha256(ssh_file_content) + OpenSSL::Digest::SHA256.hexdigest(ssh_file_content) + end +end diff --git a/app/models/issue.rb b/app/models/issue.rb index 92a454300af..fc590f9257e 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -10,6 +10,7 @@ class Issue < ActiveRecord::Base include FasterCacheKeys include RelativePositioning include CreatedAtFilterable + include TimeTrackable DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze @@ -74,19 +75,7 @@ class Issue < ActiveRecord::Base end end - def hook_attrs - assignee_ids = self.assignee_ids - - attrs = { - total_time_spent: total_time_spent, - human_total_time_spent: human_total_time_spent, - human_time_estimate: human_time_estimate, - assignee_ids: assignee_ids, - assignee_id: assignee_ids.first # This key is deprecated - } - - attributes.merge!(attrs) - end + acts_as_paranoid def self.reference_prefix '#' @@ -116,7 +105,8 @@ class Issue < ActiveRecord::Base def self.sort(method, excluded_labels: []) case method.to_s - when 'due_date_asc' then order_due_date_asc + when 'due_date' then order_due_date_asc + when 'due_date_asc' then order_due_date_asc when 'due_date_desc' then order_due_date_desc else super @@ -130,6 +120,10 @@ class Issue < ActiveRecord::Base "id DESC") end + def hook_attrs + Gitlab::HookData::IssueBuilder.new(self).build + end + # Returns a Hash of attributes to be used for Twitter card metadata def card_attributes { diff --git a/app/models/key.rb b/app/models/key.rb index 0c41e34d969..f119b15c737 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -34,6 +34,7 @@ class Key < ActiveRecord::Base value&.delete!("\n\r") value.strip! unless value.blank? write_attribute(:key, value) + @public_key = nil end def publishable_key diff --git a/app/models/legacy_diff_discussion.rb b/app/models/legacy_diff_discussion.rb index 3c1d34db5fa..80fc6304fd4 100644 --- a/app/models/legacy_diff_discussion.rb +++ b/app/models/legacy_diff_discussion.rb @@ -17,6 +17,14 @@ class LegacyDiffDiscussion < Discussion true end + def on_image? + false + end + + def on_text? + true + end + def active?(*args) return @active if @active.present? diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 8d9a30397a9..3133dc9e7eb 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -6,6 +6,7 @@ class MergeRequest < ActiveRecord::Base include Sortable include IgnorableColumn include CreatedAtFilterable + include TimeTrackable ignore_column :locked_at @@ -119,6 +120,8 @@ class MergeRequest < ActiveRecord::Base after_save :keep_around_commit + acts_as_paranoid + def self.reference_prefix '!' end @@ -179,6 +182,10 @@ class MergeRequest < ActiveRecord::Base work_in_progress?(title) ? title : "WIP: #{title}" end + def hook_attrs + Gitlab::HookData::MergeRequestBuilder.new(self).build + end + # Returns a Hash of attributes to be used for Twitter card metadata def card_attributes { @@ -392,7 +399,11 @@ class MergeRequest < ActiveRecord::Base end def merge_ongoing? - !!merge_jid && !merged? + # While the MergeRequest is locked, it should present itself as 'merge ongoing'. + # The unlocking process is handled by StuckMergeJobsWorker scheduled in Cron. + return true if locked? + + !!merge_jid && !merged? && Gitlab::SidekiqStatus.running?(merge_jid) end def closed_without_fork? @@ -403,7 +414,7 @@ class MergeRequest < ActiveRecord::Base return false unless for_fork? return true unless source_project - !source_project.forked_from?(target_project) + !source_project.in_fork_network_of?(target_project) end def reopenable? @@ -415,6 +426,8 @@ class MergeRequest < ActiveRecord::Base end def create_merge_request_diff + fetch_ref + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37435 Gitlab::GitalyClient.allow_n_plus_1_calls do merge_request_diffs.create @@ -462,6 +475,7 @@ class MergeRequest < ActiveRecord::Base return unless open? old_diff_refs = self.diff_refs + create_merge_request_diff MergeRequests::MergeRequestDiffCacheService.new.execute(self) new_diff_refs = self.diff_refs @@ -474,7 +488,7 @@ class MergeRequest < ActiveRecord::Base end def check_if_can_be_merged - return unless unchecked? + return unless unchecked? && Gitlab::Database.read_write? can_be_merged = !broken? && project.repository.can_be_merged?(diff_head_sha, target_branch) @@ -524,6 +538,14 @@ class MergeRequest < ActiveRecord::Base true end + def ff_merge_possible? + project.repository.ancestor?(target_branch_sha, diff_head_sha) + end + + def should_be_rebased? + project.ff_merge_must_be_possible? && !ff_merge_possible? + end + def can_cancel_merge_when_pipeline_succeeds?(current_user) can_be_merged_by?(current_user) || self.author == current_user end @@ -552,14 +574,20 @@ class MergeRequest < ActiveRecord::Base commits_for_notes_limit = 100 commit_ids = commit_shas.take(commits_for_notes_limit) - Note.where( - "(project_id = :target_project_id AND noteable_type = 'MergeRequest' AND noteable_id = :mr_id) OR" + - "((project_id = :source_project_id OR project_id = :target_project_id) AND noteable_type = 'Commit' AND commit_id IN (:commit_ids))", - mr_id: id, - commit_ids: commit_ids, - target_project_id: target_project_id, - source_project_id: source_project_id - ) + commit_notes = Note + .except(:order) + .where(project_id: [source_project_id, target_project_id]) + .where(noteable_type: 'Commit', commit_id: commit_ids) + + # We're using a UNION ALL here since this results in better performance + # compared to using OR statements. We're using UNION ALL since the queries + # used won't produce any duplicates (e.g. a note for a commit can't also be + # a note for an MR). + union = Gitlab::SQL::Union + .new([notes, commit_notes], remove_duplicates: false) + .to_sql + + Note.from("(#{union}) #{Note.table_name}") end alias_method :discussion_notes, :related_notes @@ -570,24 +598,6 @@ class MergeRequest < ActiveRecord::Base !discussions_to_be_resolved? end - def hook_attrs - attrs = { - source: source_project.try(:hook_attrs), - target: target_project.hook_attrs, - last_commit: nil, - work_in_progress: work_in_progress?, - total_time_spent: total_time_spent, - human_total_time_spent: human_total_time_spent, - human_time_estimate: human_time_estimate - } - - if diff_head_commit - attrs[:last_commit] = diff_head_commit.hook_attrs - end - - attributes.merge!(attrs) - end - def for_fork? target_project != source_project end @@ -672,13 +682,13 @@ class MergeRequest < ActiveRecord::Base def source_branch_exists? return false unless self.source_project - self.source_project.repository.branch_names.include?(self.source_branch) + self.source_project.repository.branch_exists?(self.source_branch) end def target_branch_exists? return false unless self.target_project - self.target_project.repository.branch_names.include?(self.target_branch) + self.target_project.repository.branch_exists?(self.target_branch) end def merge_commit_message(include_description: false) @@ -734,10 +744,9 @@ class MergeRequest < ActiveRecord::Base end def has_ci? - has_ci_integration = source_project.try(:ci_service) - uses_gitlab_ci = all_pipelines.any? + return false if has_no_commits? - (has_ci_integration || uses_gitlab_ci) && commits.any? + !!(head_pipeline_id || all_pipelines.any? || source_project&.ci_service) end def branch_missing? @@ -872,7 +881,7 @@ class MergeRequest < ActiveRecord::Base # def all_commit_shas if persisted? - column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).pluck('DISTINCT(sha)') + column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).limit(10_000).pluck('sha') serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas) (column_shas + serialised_shas).uniq diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 58050e1f438..1eda0f9cbbd 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -48,6 +48,10 @@ class MergeRequestDiff < ActiveRecord::Base # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content + MergeRequest + .where('id = ? AND COALESCE(latest_merge_request_diff_id, 0) < ?', self.merge_request_id, self.id) + .update_all(latest_merge_request_diff_id: self.id) + ensure_commit_shas save_commits save_diffs @@ -55,7 +59,6 @@ class MergeRequestDiff < ActiveRecord::Base end def ensure_commit_shas - merge_request.fetch_ref self.start_commit_sha ||= merge_request.target_branch_sha self.head_commit_sha ||= merge_request.source_branch_sha self.base_commit_sha ||= find_base_sha diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index 670b26d4ca3..b75387e236e 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -17,7 +17,9 @@ class MergeRequestDiffCommit < ActiveRecord::Base commit_hash.merge( merge_request_diff_id: merge_request_diff_id, relative_order: index, - sha: sha_attribute.type_cast_for_database(sha) + sha: sha_attribute.type_cast_for_database(sha), + authored_date: Gitlab::Database.sanitize_timestamp(commit_hash[:authored_date]), + committed_date: Gitlab::Database.sanitize_timestamp(commit_hash[:committed_date]) ) end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index e279d8dd8c5..0601a61a926 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -139,7 +139,9 @@ class Namespace < ActiveRecord::Base end def find_fork_of(project) - projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id) + return nil unless project.fork_network + + project.fork_network.find_forks_in(projects).first end def lfs_enabled? @@ -160,6 +162,13 @@ class Namespace < ActiveRecord::Base .base_and_ancestors end + # returns all ancestors upto but excluding the the given namespace + # when no namespace is given, all ancestors upto the top are returned + def ancestors_upto(top = nil) + Gitlab::GroupHierarchy.new(self.class.where(id: id)) + .ancestors(upto: top) + end + def self_and_ancestors return self.class.where(id: id) unless parent_id diff --git a/app/models/note.rb b/app/models/note.rb index f44590e2144..f9676361072 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -69,7 +69,7 @@ class Note < ActiveRecord::Base delegate :title, to: :noteable, allow_nil: true validates :note, presence: true - validates :project, presence: true, unless: :for_personal_snippet? + validates :project, presence: true, if: :for_project_noteable? # Attachments are deprecated and are handled by Markdown uploader validates :attachment, file_size: { maximum: :max_attachment_size } @@ -114,7 +114,7 @@ class Note < ActiveRecord::Base after_initialize :ensure_discussion_id before_validation :nullify_blank_type, :nullify_blank_line_code before_validation :set_discussion_id, on: :create - after_save :keep_around_commit, unless: :for_personal_snippet? + after_save :keep_around_commit, if: :for_project_noteable? after_save :expire_etag_cache after_destroy :expire_etag_cache @@ -134,14 +134,22 @@ class Note < ActiveRecord::Base Discussion.build(notes) end + # Group diff discussions by line code or file path. + # It is not needed to group by line code when comment is + # on an image. def grouped_diff_discussions(diff_refs = nil) groups = {} diff_notes.fresh.discussions.each do |discussion| - line_code = discussion.line_code_in_diffs(diff_refs) - - if line_code - discussions = groups[line_code] ||= [] + group_key = + if discussion.on_image? + discussion.file_new_path + else + discussion.line_code_in_diffs(diff_refs) + end + + if group_key + discussions = groups[group_key] ||= [] discussions << discussion end end @@ -161,7 +169,7 @@ class Note < ActiveRecord::Base end def cross_reference? - system? && SystemNoteService.cross_reference?(note) + system? && matches_cross_reference_regex? end def diff_note? @@ -200,6 +208,10 @@ class Note < ActiveRecord::Base noteable.is_a?(PersonalSnippet) end + def for_project_noteable? + !for_personal_snippet? + end + def skip_project_check? for_personal_snippet? end diff --git a/app/models/oauth_access_token.rb b/app/models/oauth_access_token.rb index b85f5dbaf2e..e8595b13d6d 100644 --- a/app/models/oauth_access_token.rb +++ b/app/models/oauth_access_token.rb @@ -1,4 +1,14 @@ class OauthAccessToken < Doorkeeper::AccessToken belongs_to :resource_owner, class_name: 'User' belongs_to :application, class_name: 'Doorkeeper::Application' + + alias_attribute :user, :resource_owner + + def scopes=(value) + if value.is_a?(Array) + super(Doorkeeper::OAuth::Scopes.from_array(value).to_s) + else + super + end + end end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 5d798247863..2e824cda525 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -16,9 +16,9 @@ class PagesDomain < ActiveRecord::Base key: Gitlab::Application.secrets.db_key_base, algorithm: 'aes-256-cbc' - after_create :update - after_save :update - after_destroy :update + after_create :update_daemon + after_save :update_daemon + after_destroy :update_daemon def to_param domain @@ -80,7 +80,7 @@ class PagesDomain < ActiveRecord::Base private - def update + def update_daemon ::Projects::UpdatePagesConfigurationService.new(project).execute end diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 1f9d712ef84..cfcb03138b7 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -17,6 +17,8 @@ class PersonalAccessToken < ActiveRecord::Base validates :scopes, presence: true validate :validate_scopes + after_initialize :set_default_scopes, if: :persisted? + def revoke! update!(revoked: true) end @@ -32,4 +34,8 @@ class PersonalAccessToken < ActiveRecord::Base errors.add :scopes, "can only contain available scopes" end end + + def set_default_scopes + self.scopes = Gitlab::Auth::DEFAULT_SCOPES if self.scopes.empty? + end end diff --git a/app/models/project.rb b/app/models/project.rb index f7221e4f3b2..3f810ee977b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -17,6 +17,7 @@ class Project < ActiveRecord::Base include ProjectFeaturesCompatibility include SelectForProjectAuthorization include Routable + include GroupDescendant extend Gitlab::ConfigHelper extend Gitlab::CurrentSettings @@ -25,7 +26,15 @@ class Project < ActiveRecord::Base NUMBER_OF_PERMITTED_BOARDS = 1 UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze - LATEST_STORAGE_VERSION = 1 + # Hashed Storage versions handle rolling out new storage to project and dependents models: + # nil: legacy + # 1: repository + # 2: attachments + LATEST_STORAGE_VERSION = 2 + HASHED_STORAGE_FEATURES = { + repository: 1, + attachments: 2 + }.freeze cache_markdown_field :description, pipeline: :description @@ -64,6 +73,7 @@ class Project < ActiveRecord::Base # Storage specific hooks after_initialize :use_hashed_storage + after_create :check_repository_absence! after_create :ensure_storage_path_exists after_save :ensure_storage_path_exists, if: :namespace_id_changed? @@ -72,6 +82,7 @@ class Project < ActiveRecord::Base attr_accessor :old_path_with_namespace attr_accessor :template_name attr_writer :pipeline_status + attr_accessor :skip_disk_validation alias_attribute :title, :name @@ -79,6 +90,8 @@ class Project < ActiveRecord::Base belongs_to :creator, class_name: 'User' belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id' belongs_to :namespace + alias_method :parent, :namespace + alias_attribute :parent_id, :namespace_id has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event' has_many :boards, before_add: :validate_board_limit @@ -115,12 +128,22 @@ class Project < ActiveRecord::Base has_one :mock_deployment_service has_one :mock_monitoring_service has_one :microsoft_teams_service + has_one :packagist_service + # TODO: replace these relations with the fork network versions has_one :forked_project_link, foreign_key: "forked_to_project_id" has_one :forked_from_project, through: :forked_project_link has_many :forked_project_links, foreign_key: "forked_from_project_id" has_many :forks, through: :forked_project_links, source: :forked_to_project + # TODO: replace these relations with the fork network versions + + has_one :root_of_fork_network, + foreign_key: 'root_project_id', + inverse_of: :root_project, + class_name: 'ForkNetwork' + has_one :fork_network_member + has_one :fork_network, through: :fork_network_member # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id' @@ -163,6 +186,7 @@ class Project < ActiveRecord::Base has_one :import_data, class_name: 'ProjectImportData', inverse_of: :project, autosave: true has_one :project_feature, inverse_of: :project has_one :statistics, class_name: 'ProjectStatistics' + has_one :cluster, class_name: 'Gcp::Cluster', inverse_of: :project # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy @@ -177,6 +201,7 @@ class Project < ActiveRecord::Base # bulk that doesn't involve loading the rows into memory. As a result we're # still using `dependent: :destroy` here. has_many :builds, class_name: 'Ci::Build', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' has_many :runner_projects, class_name: 'Ci::RunnerProject' has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' @@ -227,7 +252,7 @@ class Project < ActiveRecord::Base validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?] validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create - validate :can_create_repository?, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? } + validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } validate :avatar_type, if: ->(project) { project.avatar.present? && project.avatar_changed? } validates :avatar, file_size: { maximum: 200.kilobytes.to_i } @@ -245,6 +270,9 @@ class Project < ActiveRecord::Base scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } + scope :with_hashed_storage, -> { where('storage_version >= 1') } + scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) } + scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) } scope :sorted_by_stars, -> { reorder('projects.star_count DESC') } @@ -463,6 +491,13 @@ class Project < ActiveRecord::Base end end + # returns all ancestor-groups upto but excluding the given namespace + # when no namespace is given, all ancestors upto the top are returned + def ancestors_upto(top = nil) + Gitlab::GroupHierarchy.new(Group.where(id: namespace_id)) + .base_and_ancestors(upto: top) + end + def lfs_enabled? return namespace.lfs_enabled? if self[:lfs_enabled].nil? @@ -514,6 +549,10 @@ class Project < ActiveRecord::Base repository.commit(ref) end + def commit_by(oid:) + repository.commit_by(oid: oid) + end + # ref can't be HEAD, can only be branch/tag name or SHA def latest_successful_builds_for(ref = default_branch) latest_pipeline = pipelines.latest_successful_for(ref) @@ -527,7 +566,7 @@ class Project < ActiveRecord::Base def merge_base_commit(first_commit_id, second_commit_id) sha = repository.merge_base(first_commit_id, second_commit_id) - repository.commit(sha) if sha + commit_by(oid: sha) if sha end def saved? @@ -808,7 +847,7 @@ class Project < ActiveRecord::Base end def cache_has_external_issue_tracker - update_column(:has_external_issue_tracker, services.external_issue_trackers.any?) + update_column(:has_external_issue_tracker, services.external_issue_trackers.any?) if Gitlab::Database.read_write? end def has_wiki? @@ -828,7 +867,7 @@ class Project < ActiveRecord::Base end def cache_has_external_wiki - update_column(:has_external_wiki, services.external_wikis.any?) + update_column(:has_external_wiki, services.external_wikis.any?) if Gitlab::Database.read_write? end def find_or_initialize_services(exceptions: []) @@ -993,9 +1032,18 @@ class Project < ActiveRecord::Base end def forked? + return true if fork_network && fork_network.root_project != self + + # TODO: Use only the above conditional using the `fork_network` + # This is the old conditional that looks at the `forked_project_link`, we + # fall back to this while we're migrating the new models !(forked_project_link.nil? || forked_project_link.forked_from_project.nil?) end + def fork_source + forked_from_project || fork_network&.root_project + end + def personal? !group end @@ -1015,24 +1063,29 @@ class Project < ActiveRecord::Base end # Check if repository already exists on disk - def can_create_repository? + def check_repository_path_availability + return true if skip_disk_validation return false unless repository_storage_path expires_full_path_cache # we need to clear cache to validate renames correctly - if gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git") + # Check if repository with same path already exists on disk we can + # skip this for the hashed storage because the path does not change + if legacy_storage? && repository_with_same_path_already_exists? errors.add(:base, 'There is already a repository with that name on disk') return false end true + rescue GRPC::Internal # if the path is too long + false end def create_repository(force: false) # Forked import is handled asynchronously return if forked? && !force - if gitlab_shell.add_repository(repository_storage_path, disk_path) + if gitlab_shell.add_repository(repository_storage, disk_path) repository.after_create true else @@ -1043,6 +1096,7 @@ class Project < ActiveRecord::Base def hook_attrs(backward: true) attrs = { + id: id, name: name, description: description, web_url: web_url, @@ -1107,8 +1161,19 @@ class Project < ActiveRecord::Base end end - def forked_from?(project) - forked? && project == forked_from_project + def forked_from?(other_project) + forked? && forked_from_project == other_project + end + + def in_fork_network_of?(other_project) + # TODO: Remove this in a next release when all fork_networks are populated + # This makes sure all MergeRequests remain valid while the projects don't + # have a fork_network yet. + return true if forked_from?(other_project) + + return false if fork_network.nil? || other_project.fork_network.nil? + + fork_network == other_project.fork_network end def origin_merge_requests @@ -1225,7 +1290,7 @@ class Project < ActiveRecord::Base # self.forked_from_project will be nil before the project is saved, so # we need to go through the relation - original_project = forked_project_link.forked_from_project + original_project = forked_project_link&.forked_from_project return true unless original_project level <= original_project.visibility_level @@ -1343,6 +1408,19 @@ class Project < ActiveRecord::Base end end + def after_rename_repo + path_before_change = previous_changes['path'].first + + # We need to check if project had been rolled out to move resource to hashed storage or not and decide + # if we need execute any take action or no-op. + + unless hashed_storage?(:attachments) + Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) + end + + Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) + end + def rename_repo_notify! send_move_instructions(full_path_was) expires_full_path_cache @@ -1353,13 +1431,6 @@ class Project < ActiveRecord::Base reload_repository! end - def after_rename_repo - path_before_change = previous_changes['path'].first - - Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) - Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path) - end - def running_or_pending_build_count(force: false) Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do builds.running_or_pending.count(:all) @@ -1421,7 +1492,8 @@ class Project < ActiveRecord::Base { key: 'CI_PROJECT_PATH', value: full_path, public: true }, { key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug, public: true }, { key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path, public: true }, - { key: 'CI_PROJECT_URL', value: web_url, public: true } + { key: 'CI_PROJECT_URL', value: web_url, public: true }, + { key: 'CI_PROJECT_VISIBILITY', value: Gitlab::VisibilityLevel.string_level(visibility_level), public: true } ] end @@ -1512,10 +1584,6 @@ class Project < ActiveRecord::Base map.public_path_for_source_path(path) end - def parent - namespace - end - def parent_changed? namespace_id_changed? end @@ -1550,18 +1618,81 @@ class Project < ActiveRecord::Base end def legacy_storage? - self.storage_version.nil? + [nil, 0].include?(self.storage_version) + end + + # Check if Hashed Storage is enabled for the project with at least informed feature rolled out + # + # @param [Symbol] feature that needs to be rolled out for the project (:repository, :attachments) + def hashed_storage?(feature) + raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature) + + self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature] end def renamed? persisted? && path_changed? end + def merge_method + if self.merge_requests_ff_only_enabled + :ff + elsif self.merge_requests_rebase_enabled + :rebase_merge + else + :merge + end + end + + def merge_method=(method) + case method.to_s + when "ff" + self.merge_requests_ff_only_enabled = true + self.merge_requests_rebase_enabled = true + when "rebase_merge" + self.merge_requests_ff_only_enabled = false + self.merge_requests_rebase_enabled = true + when "merge" + self.merge_requests_ff_only_enabled = false + self.merge_requests_rebase_enabled = false + end + end + + def ff_merge_must_be_possible? + self.merge_requests_ff_only_enabled || self.merge_requests_rebase_enabled + end + + def migrate_to_hashed_storage! + return if hashed_storage?(:repository) + + update!(repository_read_only: true) + + if repo_reference_count > 0 || wiki_reference_count > 0 + ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id) + else + ProjectMigrateHashedStorageWorker.perform_async(id) + end + end + + def storage_version=(value) + super + + @storage = nil if storage_version_changed? + end + + def gl_repository(is_wiki:) + Gitlab::GlRepository.gl_repository(self, is_wiki) + end + + def reference_counter(wiki: false) + Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki)) + end + private def storage @storage ||= - if self.storage_version && self.storage_version >= 1 + if hashed_storage?(:repository) Storage::HashedProject.new(self) else Storage::LegacyProject.new(self) @@ -1574,6 +1705,27 @@ class Project < ActiveRecord::Base end end + def repo_reference_count + reference_counter.value + end + + def wiki_reference_count + reference_counter(wiki: true).value + end + + def check_repository_absence! + return if skip_disk_validation + + if repository_storage_path.blank? || repository_with_same_path_already_exists? + errors.add(:base, 'There is already a repository with that name on disk') + throw :abort + end + end + + def repository_with_same_path_already_exists? + gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git") + end + # set last_activity_at to the same as created_at def set_last_activity_at update_column(:last_activity_at, self.created_at) diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index e2ad586aea7..22a65b5145e 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -3,6 +3,7 @@ require 'slack-notifier' module ChatMessage class BaseMessage attr_reader :markdown + attr_reader :user_full_name attr_reader :user_name attr_reader :user_avatar attr_reader :project_name @@ -12,10 +13,19 @@ module ChatMessage @markdown = params[:markdown] || false @project_name = params.dig(:project, :path_with_namespace) || params[:project_name] @project_url = params.dig(:project, :web_url) || params[:project_url] + @user_full_name = params.dig(:user, :name) || params[:user_full_name] @user_name = params.dig(:user, :username) || params[:user_name] @user_avatar = params.dig(:user, :avatar_url) || params[:user_avatar] end + def user_combined_name + if user_full_name.present? + "#{user_full_name} (#{user_name})" + else + user_name + end + end + def pretext return message if markdown diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb index 4b9a2b1e1f3..1327b075858 100644 --- a/app/models/project_services/chat_message/issue_message.rb +++ b/app/models/project_services/chat_message/issue_message.rb @@ -29,7 +29,7 @@ module ChatMessage def activity { - title: "Issue #{state} by #{user_name}", + title: "Issue #{state} by #{user_combined_name}", subtitle: "in #{project_link}", text: issue_link, image: user_avatar @@ -40,9 +40,9 @@ module ChatMessage def message if state == 'opened' - "[#{project_link}] Issue #{state} by #{user_name}" + "[#{project_link}] Issue #{state} by #{user_combined_name}" else - "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}" + "[#{project_link}] Issue #{issue_link} #{state} by #{user_combined_name}" end end diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index 7d0de81cdf0..f412b6833d9 100644 --- a/app/models/project_services/chat_message/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -24,7 +24,7 @@ module ChatMessage def activity { - title: "Merge Request #{state} by #{user_name}", + title: "Merge Request #{state} by #{user_combined_name}", subtitle: "in #{project_link}", text: merge_request_link, image: user_avatar @@ -46,7 +46,7 @@ module ChatMessage end def merge_request_message - "#{user_name} #{state} #{merge_request_link} in #{project_link}: #{title}" + "#{user_combined_name} #{state} #{merge_request_link} in #{project_link}: #{title}" end def merge_request_link diff --git a/app/models/project_services/chat_message/note_message.rb b/app/models/project_services/chat_message/note_message.rb index 2da4c244229..7f9486132e6 100644 --- a/app/models/project_services/chat_message/note_message.rb +++ b/app/models/project_services/chat_message/note_message.rb @@ -32,7 +32,7 @@ module ChatMessage def activity { - title: "#{user_name} #{link('commented on ' + target, note_url)}", + title: "#{user_combined_name} #{link('commented on ' + target, note_url)}", subtitle: "in #{project_link}", text: formatted_title, image: user_avatar @@ -42,7 +42,7 @@ module ChatMessage private def message - "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{formatted_title}*" + "#{user_combined_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{formatted_title}*" end def format_title(title) diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index d63d4ec2b12..2135122278a 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -9,7 +9,7 @@ module ChatMessage def initialize(data) super - @user_name = data.dig(:user, :name) || 'API' + @user_name = data.dig(:user, :username) || 'API' pipeline_attributes = data[:object_attributes] @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' @@ -35,7 +35,7 @@ module ChatMessage def activity { - title: "Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_name} #{humanized_status}", + title: "Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status}", subtitle: "in #{project_link}", text: "in #{pretty_duration(duration)}", image: user_avatar || '' @@ -45,7 +45,7 @@ module ChatMessage private def message - "#{project_link}: Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_name} #{humanized_status} in #{pretty_duration(duration)}" + "#{project_link}: Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status} in #{pretty_duration(duration)}" end def humanized_status diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb index c52dd6ef8ef..8d599c5f116 100644 --- a/app/models/project_services/chat_message/push_message.rb +++ b/app/models/project_services/chat_message/push_message.rb @@ -33,7 +33,7 @@ module ChatMessage end { - title: "#{user_name} #{action} #{ref_type}", + title: "#{user_combined_name} #{action} #{ref_type}", subtitle: "in #{project_link}", text: compare_link, image: user_avatar @@ -57,15 +57,15 @@ module ChatMessage end def new_branch_message - "#{user_name} pushed new #{ref_type} #{branch_link} to #{project_link}" + "#{user_combined_name} pushed new #{ref_type} #{branch_link} to #{project_link}" end def removed_branch_message - "#{user_name} removed #{ref_type} #{ref} from #{project_link}" + "#{user_combined_name} removed #{ref_type} #{ref} from #{project_link}" end def push_message - "#{user_name} pushed to #{ref_type} #{branch_link} of #{project_link} (#{compare_link})" + "#{user_combined_name} pushed to #{ref_type} #{branch_link} of #{project_link} (#{compare_link})" end def commit_messages diff --git a/app/models/project_services/chat_message/wiki_page_message.rb b/app/models/project_services/chat_message/wiki_page_message.rb index a139a8ee727..d84b80f2de2 100644 --- a/app/models/project_services/chat_message/wiki_page_message.rb +++ b/app/models/project_services/chat_message/wiki_page_message.rb @@ -31,7 +31,7 @@ module ChatMessage def activity { - title: "#{user_name} #{action} #{wiki_page_link}", + title: "#{user_combined_name} #{action} #{wiki_page_link}", subtitle: "in #{project_link}", text: title, image: user_avatar @@ -41,7 +41,7 @@ module ChatMessage private def message - "#{user_name} #{action} #{wiki_page_link} in #{project_link}: *#{title}*" + "#{user_combined_name} #{action} #{wiki_page_link} in #{project_link}: *#{title}*" end def description_message diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 9ee3a533c1e..b487378edd2 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -3,6 +3,8 @@ class JiraService < IssueTrackerService validates :url, url: true, presence: true, if: :activated? validates :api_url, url: true, allow_blank: true + validates :username, presence: true, if: :activated? + validates :password, presence: true, if: :activated? prop_accessor :username, :password, :url, :api_url, :jira_issue_transition_id, :title, :description diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 8ba07173c74..5c0b3338a62 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -153,7 +153,10 @@ class KubernetesService < DeploymentService end def default_namespace - "#{project.path}-#{project.id}" if project.present? + return unless project + + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') end def build_kubeclient!(api_path: 'api', api_version: 'v1') diff --git a/app/models/project_services/packagist_service.rb b/app/models/project_services/packagist_service.rb new file mode 100644 index 00000000000..f68a0c1a3c3 --- /dev/null +++ b/app/models/project_services/packagist_service.rb @@ -0,0 +1,65 @@ +class PackagistService < Service + include HTTParty + + prop_accessor :username, :token, :server + + validates :username, presence: true, if: :activated? + validates :token, presence: true, if: :activated? + + default_value_for :push_events, true + default_value_for :tag_push_events, true + + after_save :compose_service_hook, if: :activated? + + def title + 'Packagist' + end + + def description + 'Update your project on Packagist, the main Composer repository' + end + + def self.to_param + 'packagist' + end + + def fields + [ + { type: 'text', name: 'username', placeholder: '', required: true }, + { type: 'text', name: 'token', placeholder: '', required: true }, + { type: 'text', name: 'server', placeholder: 'https://packagist.org', required: false } + ] + end + + def self.supported_events + %w(push merge_request tag_push) + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + service_hook.execute(data) + end + + def test(data) + begin + result = execute(data) + return { success: false, result: result[:message] } if result[:http_status] != 202 + rescue StandardError => error + return { success: false, result: error } + end + + { success: true, result: result[:message] } + end + + def compose_service_hook + hook = service_hook || build_service_hook + hook.url = hook_url + hook.save + end + + def hook_url + base_url = server.present? ? server : 'https://packagist.org' + "#{base_url}/api/update-package?username=#{username}&apiToken=#{token}" + end +end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 698fdf7a20c..43de6809178 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -54,12 +54,15 @@ class ProjectWiki [Gitlab.config.gitlab.relative_url_root, '/', @project.full_path, '/wikis'].join('') end - # Returns the Gollum::Wiki object. + # Returns the Gitlab::Git::Wiki object. def wiki @wiki ||= begin - Gollum::Wiki.new(path_to_repo) - rescue Rugged::OSError - create_repo! + gl_repository = Gitlab::GlRepository.gl_repository(project, true) + raw_repository = Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', gl_repository) + + create_repo!(raw_repository) unless raw_repository.exists? + + Gitlab::Git::Wiki.new(raw_repository) end end @@ -86,20 +89,14 @@ class ProjectWiki # Returns an initialized WikiPage instance or nil def find_page(title, version = nil) page_title, page_dir = page_title_and_dir(title) - if page = wiki.page(page_title, version, page_dir) + + if page = wiki.page(title: page_title, version: version, dir: page_dir) WikiPage.new(self, page, true) - else - nil end end - def find_file(name, version = nil, try_on_disk = true) - version = wiki.ref if version.nil? # Gollum::Wiki#file ? - if wiki_file = wiki.file(name, version, try_on_disk) - wiki_file - else - nil - end + def find_file(name, version = nil) + wiki.file(name, version) end def create_page(title, content, format = :markdown, message = nil) @@ -108,7 +105,7 @@ class ProjectWiki wiki.write_page(title, format.to_sym, content, commit) update_project_activity - rescue Gollum::DuplicatePageError => e + rescue Gitlab::Git::Wiki::DuplicatePageError => e @error_message = "Duplicate page: #{e.message}" return false end @@ -116,13 +113,13 @@ class ProjectWiki def update_page(page, content:, title: nil, format: :markdown, message: nil) commit = commit_details(:updated, message, page.title) - wiki.update_page(page, title || page.name, format.to_sym, content, commit) + wiki.update_page(page.path, title || page.name, format.to_sym, content, commit) update_project_activity end def delete_page(page, message = nil) - wiki.delete_page(page, commit_details(:deleted, message, page.title)) + wiki.delete_page(page.path, commit_details(:deleted, message, page.title)) update_project_activity end @@ -138,27 +135,15 @@ class ProjectWiki end def repository - @repository ||= Repository.new(full_path, @project, disk_path: disk_path) + @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true) end def default_branch wiki.class.default_ref end - def create_repo! - if init_repo(disk_path) - wiki = Gollum::Wiki.new(path_to_repo) - else - raise CouldNotCreateWikiError - end - - repository.after_create - - wiki - end - def ensure_repository - create_repo! unless repository_exists? + raise CouldNotCreateWikiError unless wiki.repository_exists? end def hook_attrs @@ -173,24 +158,24 @@ class ProjectWiki private - def init_repo(disk_path) - gitlab_shell.add_repository(project.repository_storage_path, disk_path) + def create_repo!(raw_repository) + gitlab_shell.add_repository(project.repository_storage, disk_path) + + raise CouldNotCreateWikiError unless raw_repository.exists? + + repository.after_create end def commit_details(action, message = nil, title = nil) commit_message = message || default_message(action, title) - { email: @user.email, name: @user.name, message: commit_message } + Gitlab::Git::Wiki::CommitDetails.new(@user.name, @user.email, commit_message) end def default_message(action, title) "#{@user.username} #{action} page: #{title}" end - def path_to_repo - @path_to_repo ||= File.join(project.repository_storage_path, "#{disk_path}.git") - end - def update_project_activity @project.touch(:last_activity_at, :last_repository_updated_at) end diff --git a/app/models/repository.rb b/app/models/repository.rb index 90cede9d3d4..69cddb36b2e 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -15,9 +15,8 @@ class Repository ].freeze include Gitlab::ShellAdapter - include RepositoryMirroring - attr_accessor :full_path, :disk_path, :project + attr_accessor :full_path, :disk_path, :project, :is_wiki delegate :ref_name_for_sha, to: :raw_repository @@ -34,7 +33,11 @@ class Repository CACHED_METHODS = %i(size commit_count rendered_readme contribution_guide changelog license_blob license_key gitignore koding_yml gitlab_ci_yml branch_names tag_names branch_count - tag_count avatar exists? empty? root_ref).freeze + tag_count avatar exists? empty? root_ref has_visible_content? + issue_template_names merge_request_template_names).freeze + + # Methods that use cache_method but only memoize the value + MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze # Certain method caches should be refreshed when certain types of files are # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to @@ -47,7 +50,9 @@ class Repository gitignore: :gitignore, koding: :koding_yml, gitlab_ci: :gitlab_ci_yml, - avatar: :avatar + avatar: :avatar, + issue_template: :issue_template_names, + merge_request_template: :merge_request_template_names }.freeze # Wraps around the given method and caches its output in Redis and an instance @@ -66,10 +71,12 @@ class Repository end end - def initialize(full_path, project, disk_path: nil) + def initialize(full_path, project, disk_path: nil, is_wiki: false) @full_path = full_path @disk_path = disk_path || full_path @project = project + @commit_cache = {} + @is_wiki = is_wiki end def ==(other) @@ -91,30 +98,23 @@ class Repository ) end - # we need to have this method here because it is not cached in ::Git and - # the method is called multiple times for every request - def has_visible_content? - branch_count > 0 - end - def inspect "#<#{self.class.name}:#{@disk_path}>" end def commit(ref = 'HEAD') return nil unless exists? + return ref if ref.is_a?(::Commit) - commit = - if ref.is_a?(Gitlab::Git::Commit) - ref - else - Gitlab::Git::Commit.find(raw_repository, ref) - end + find_commit(ref) + end - commit = ::Commit.new(commit, @project) if commit - commit - rescue Rugged::OdbError, Rugged::TreeError - nil + # Finding a commit by the passed SHA + # Also takes care of caching, based on the SHA + def commit_by(oid:) + return @commit_cache[oid] if @commit_cache.key?(oid) + + @commit_cache[oid] = find_commit(oid) end def commits(ref, path: nil, limit: nil, offset: nil, skip_merges: false, after: nil, before: nil) @@ -231,7 +231,7 @@ class Repository # branches or tags, but we want to keep some of these commits around, for # example if they have comments or CI builds. def keep_around(sha) - return unless sha && commit(sha) + return unless sha && commit_by(oid: sha) return if kept_around?(sha) @@ -275,7 +275,7 @@ class Repository end def expire_branches_cache - expire_method_caches(%i(branch_names branch_count)) + expire_method_caches(%i(branch_names branch_count has_visible_content?)) @local_branches = nil @branch_exists_memo = nil end @@ -346,7 +346,7 @@ class Repository def expire_emptiness_caches return unless empty? - expire_method_caches(%i(empty?)) + expire_method_caches(%i(empty? has_visible_content?)) end def lookup_cache @@ -468,9 +468,7 @@ class Repository end def blob_at(sha, path) - unless Gitlab::Git.blank_ref?(sha) - Blob.decorate(Gitlab::Git::Blob.find(self, sha, path), project) - end + Blob.decorate(raw_repository.blob_at(sha, path), project) rescue Gitlab::Git::Repository::NoRepository nil end @@ -489,13 +487,7 @@ class Repository def exists? return false unless full_path - Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| - if enabled - raw_repository.exists? - else - refs_directory_exists? - end - end + raw_repository.exists? end cache_method :exists? @@ -529,17 +521,31 @@ class Repository delegate :tag_names, to: :raw_repository cache_method :tag_names, fallback: [] - delegate :branch_count, :tag_count, to: :raw_repository + delegate :branch_count, :tag_count, :has_visible_content?, to: :raw_repository cache_method :branch_count, fallback: 0 cache_method :tag_count, fallback: 0 + cache_method :has_visible_content?, fallback: false def avatar - if tree = file_on_head(:avatar) - tree.path + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38327 + Gitlab::GitalyClient.allow_n_plus_1_calls do + if tree = file_on_head(:avatar) + tree.path + end end end cache_method :avatar + def issue_template_names + Gitlab::Template::IssueTemplate.dropdown_names(project) + end + cache_method :issue_template_names, fallback: [] + + def merge_request_template_names + Gitlab::Template::MergeRequestTemplate.dropdown_names(project) + end + cache_method :merge_request_template_names, fallback: [] + def readme if readme = tree(:head)&.readme ReadmeBlob.new(readme, self) @@ -855,6 +861,15 @@ class Repository end end + def ff_merge(user, source, target_branch, merge_request: nil) + their_commit_id = commit(source)&.id + raise 'Invalid merge source' if their_commit_id.nil? + + merge_request&.update(in_progress_merge_commit_sha: their_commit_id) + + with_cache_hooks { raw.ff_merge(user, their_commit_id, target_branch) } + end + def revert( user, commit, branch_name, message, start_branch_name: nil, start_project: project) @@ -887,26 +902,27 @@ class Repository end end - def resolve_conflicts(user, branch_name, params) - with_branch(user, branch_name) do - committer = user_to_committer(user) + def merged_to_root_ref?(branch_or_name, pre_loaded_merged_branches = nil) + branch = Gitlab::Git::Branch.find(self, branch_or_name) - create_commit(params.merge(author: committer, committer: committer)) - end - end - - def merged_to_root_ref?(branch_name) - branch_commit = commit(branch_name) - root_ref_commit = commit(root_ref) + if branch + root_ref_sha = commit(root_ref).sha + same_head = branch.target == root_ref_sha + merged = + if pre_loaded_merged_branches + pre_loaded_merged_branches.include?(branch.name) + else + ancestor?(branch.target, root_ref_sha) + end - if branch_commit - same_head = branch_commit.id == root_ref_commit.id - !same_head && ancestor?(branch_commit.id, root_ref_commit.id) + !same_head && merged else nil end end + delegate :merged_branch_names, to: :raw_repository + def merge_base(first_commit_id, second_commit_id) first_commit_id = commit(first_commit_id).try(:id) || first_commit_id second_commit_id = commit(second_commit_id).try(:id) || second_commit_id @@ -949,21 +965,8 @@ class Repository run_git(args).first.lines.map(&:strip) end - def add_remote(name, url) - raw_repository.remote_add(name, url) - rescue Rugged::ConfigError - raw_repository.remote_update(name, url: url) - end - - def remove_remote(name) - raw_repository.remote_delete(name) - true - rescue Rugged::ConfigError - false - end - - def fetch_remote(remote, forced: false, no_tags: false) - gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags) + def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false) + gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) end def fetch_source_branch(source_repository, source_branch, local_ref) @@ -975,7 +978,7 @@ class Repository end def create_ref(ref, ref_path) - fetch_ref(path_to_repo, ref, ref_path) + raw_repository.write_ref(ref_path, ref) end def ls_files(ref) @@ -1014,6 +1017,10 @@ class Repository if instance_variable_defined?(ivar) instance_variable_get(ivar) else + # If the repository doesn't exist and a fallback was specified we return + # that value inmediately. This saves us Rugged/gRPC invocations. + return fallback unless fallback.nil? || exists? + begin value = if memoize_only @@ -1023,8 +1030,9 @@ class Repository end instance_variable_set(ivar, value) rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository - # if e.g. HEAD or the entire repository doesn't exist we want to - # gracefully handle this and not cache anything. + # Even if the above `#exists?` check passes these errors might still + # occur (for example because of a non-existing HEAD). We want to + # gracefully handle this and not cache anything fallback end end @@ -1052,6 +1060,18 @@ class Repository private + # TODO Generice finder, later split this on finders by Ref or Oid + # gitlab-org/gitlab-ce#39239 + def find_commit(oid_or_ref) + commit = if oid_or_ref.is_a?(Gitlab::Git::Commit) + oid_or_ref + else + Gitlab::Git::Commit.find(raw_repository, oid_or_ref) + end + + ::Commit.new(commit, @project) if commit + end + def blob_data_at(sha, path) blob = blob_at(sha, path) return unless blob @@ -1060,12 +1080,6 @@ class Repository blob.data end - def refs_directory_exists? - circuit_breaker.perform do - File.exist?(File.join(path_to_repo, 'refs')) - end - end - def cache # TODO: should we use UUIDs here? We could move repositories without clearing this cache @cache ||= RepositoryCache.new(full_path, @project.id) @@ -1096,17 +1110,17 @@ class Repository def last_commit_for_path_by_gitaly(sha, path) c = raw_repository.gitaly_commit_client.last_commit_for_path(sha, path) - commit(c) + commit_by(oid: c) end def last_commit_for_path_by_rugged(sha, path) sha = last_commit_id_for_path_by_shelling_out(sha, path) - commit(sha) + commit_by(oid: sha) end def last_commit_id_for_path_by_shelling_out(sha, path) args = %W(rev-list --max-count=1 #{sha} -- #{path}) - run_git(args).first.strip + raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip end def repository_storage_path @@ -1114,11 +1128,7 @@ class Repository end def initialize_raw_repository - Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false)) - end - - def circuit_breaker - @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(project.repository_storage) + Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki)) end def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index 298569cb7a6..6e311806be1 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -53,13 +53,17 @@ class SentNotification < ActiveRecord::Base end def unsubscribable? - !for_commit? + !(for_commit? || for_snippet?) end def for_commit? noteable_type == "Commit" end + def for_snippet? + noteable_type.end_with?('Snippet') + end + def noteable if for_commit? project.commit(commit_id) rescue nil diff --git a/app/models/service.rb b/app/models/service.rb index 6b64079215f..fdd2605e3e3 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -238,6 +238,7 @@ class Service < ActiveRecord::Base kubernetes mattermost_slash_commands mattermost + packagist pipelines_email pivotaltracker prometheus diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index fae1b64961a..f025f40994e 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -4,6 +4,7 @@ module Storage delegate :gitlab_shell, :repository_storage_path, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze + STORAGE_VERSION = 1 def initialize(project) @project = project diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 0b33e45473b..1f9f8d7286b 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -2,7 +2,7 @@ class SystemNoteMetadata < ActiveRecord::Base ICON_TYPES = %w[ commit description merge confidential visible label assignee cross_reference title time_tracking branch milestone discussion task moved - opened closed merged duplicate + opened closed merged duplicate locked unlocked outdated ].freeze diff --git a/app/models/user.rb b/app/models/user.rb index 09c9b3250eb..bcda4564595 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -21,8 +21,8 @@ class User < ActiveRecord::Base ignore_column :external_email ignore_column :email_provider + ignore_column :authentication_token - add_authentication_token_field :authentication_token add_authentication_token_field :incoming_email_token add_authentication_token_field :rss_token @@ -60,7 +60,7 @@ class User < ActiveRecord::Base lease = Gitlab::ExclusiveLease.new("user_update_tracked_fields:#{id}", timeout: 1.hour.to_i) return unless lease.try_obtain - Users::UpdateService.new(self).execute(validate: false) + Users::UpdateService.new(self, user: self).execute(validate: false) end attr_accessor :force_random_password @@ -130,6 +130,8 @@ class User < ActiveRecord::Base has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent + has_many :custom_attributes, class_name: 'UserCustomAttribute' + # # Validations # @@ -161,15 +163,17 @@ class User < ActiveRecord::Base before_validation :sanitize_attrs before_validation :set_notification_email, if: :email_changed? before_validation :set_public_email, if: :public_email_changed? - - after_update :update_emails_with_primary_email, if: :email_changed? - before_save :ensure_authentication_token, :ensure_incoming_email_token + before_save :ensure_incoming_email_token before_save :ensure_user_rights_and_limits, if: :external_changed? before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) } + before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? } after_save :ensure_namespace_correct + after_update :username_changed_hook, if: :username_changed? + after_destroy :post_destroy_hook + after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') } after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') } + after_initialize :set_projects_limit - after_destroy :post_destroy_hook # User's Layout preference enum layout: [:fixed, :fluid] @@ -179,15 +183,8 @@ class User < ActiveRecord::Base enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos] # User's Project preference - # - # Note: When adding an option, it MUST go on the end of the hash with a - # number higher than the current max. We cannot move options and/or change - # their numbers. - # - # We skip 0 because this was used by an option that has since been removed. - enum project_view: { activity: 1, files: 2 } - - alias_attribute :private_token, :authentication_token + # Note: When adding an option, it MUST go on the end of the array. + enum project_view: [:readme, :activity, :files] delegate :path, to: :namespace, allow_nil: true, prefix: true @@ -456,6 +453,14 @@ class User < ActiveRecord::Base reset_password_sent_at.present? && reset_password_sent_at >= 1.minute.ago end + def remember_me! + super if ::Gitlab::Database.read_write? + end + + def forget_me! + super if ::Gitlab::Database.read_write? + end + def disable_two_factor! transaction do update_attributes( @@ -523,12 +528,24 @@ class User < ActiveRecord::Base errors.add(:public_email, "is not an email you own") unless all_emails.include?(public_email) end + # see if the new email is already a verified secondary email + def check_for_verified_email + skip_reconfirmation! if emails.confirmed.where(email: self.email).any? + end + + # Note: the use of the Emails services will cause `saves` on the user object, running + # through the callbacks again and can have side effects, such as the `previous_changes` + # hash and `_was` variables getting munged. + # By using an `after_commit` instead of `after_update`, we avoid the recursive callback + # scenario, though it then requires us to use the `previous_changes` hash def update_emails_with_primary_email + previous_email = previous_changes[:email][0] # grab this before the DestroyService is called primary_email_record = emails.find_by(email: email) - if primary_email_record - Emails::DestroyService.new(self, email: email).execute - Emails::CreateService.new(self, email: email_was).execute - end + Emails::DestroyService.new(self, user: self).execute(primary_email_record) if primary_email_record + + # the original primary email was confirmed, and we want that to carry over. We don't + # have access to the original confirmation values at this point, so just set confirmed_at + Emails::CreateService.new(self, user: self, email: previous_email).execute(confirmed_at: confirmed_at) end def update_invalid_gpg_signatures @@ -639,6 +656,10 @@ class User < ActiveRecord::Base Ability.allowed?(self, action, subject) end + def confirm_deletion_with_password? + !password_automatically_set? && allow_password_authentication? + end + def first_name name.split.first unless name.blank? end @@ -678,19 +699,15 @@ class User < ActiveRecord::Base end def fork_of(project) - links = ForkedProjectLink.where( - forked_from_project_id: project, - forked_to_project_id: personal_projects.unscope(:order) - ) - if links.any? - links.first.forked_to_project - else - nil - end + namespace.find_fork_of(project) end def ldap_user? - identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"]) + if identities.loaded? + identities.find { |identity| identity.provider.start_with?('ldap') && !identity.extern_uid.nil? } + else + identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"]) + end end def ldap_identity @@ -810,6 +827,10 @@ class User < ActiveRecord::Base avatar_path(args) || GravatarService.new.execute(email, size, scale, username: username) end + def primary_email_verified? + confirmed? && !temp_oauth_email? + end + def all_emails all_emails = [] all_emails << email unless temp_oauth_email? @@ -817,6 +838,18 @@ class User < ActiveRecord::Base all_emails end + def verified_emails + verified_emails = [] + verified_emails << email if primary_email_verified? + verified_emails.concat(emails.confirmed.pluck(:email)) + verified_emails + end + + def verified_email?(check_email) + downcased = check_email.downcase + email == downcased ? primary_email_verified? : emails.confirmed.where(email: downcased).exists? + end + def hook_attrs { name: name, @@ -839,6 +872,10 @@ class User < ActiveRecord::Base end end + def username_changed_hook + system_hook_service.execute_hooks_for(self, :rename) + end + def post_destroy_hook log_info("User \"#{name}\" (#{email}) was removed") system_hook_service.execute_hooks_for(self, :destroy) @@ -1000,7 +1037,7 @@ class User < ActiveRecord::Base if attempts_exceeded? lock_access! unless access_locked? else - Users::UpdateService.new(self).execute(validate: false) + Users::UpdateService.new(self, user: self).execute(validate: false) end end @@ -1041,10 +1078,6 @@ class User < ActiveRecord::Base ensure_rss_token! end - def verified_email?(email) - self.email == email - end - def sync_attribute?(attribute) return true if ldap_user? && attribute == :email @@ -1061,6 +1094,12 @@ class User < ActiveRecord::Base user_synced_attributes_metadata&.read_only?(attribute) end + # override, from Devise + def lock_access! + Gitlab::AppLogger.info("Account Locked: username=#{username}") + super + end + protected # override, from Devise::Validatable @@ -1186,7 +1225,7 @@ class User < ActiveRecord::Base &creation_block ) - Users::UpdateService.new(user).execute(validate: false) + Users::UpdateService.new(user, user: user).execute(validate: false) user ensure Gitlab::ExclusiveLease.cancel(lease_key, uuid) diff --git a/app/models/user_custom_attribute.rb b/app/models/user_custom_attribute.rb new file mode 100644 index 00000000000..eff25b31f9b --- /dev/null +++ b/app/models/user_custom_attribute.rb @@ -0,0 +1,6 @@ +class UserCustomAttribute < ActiveRecord::Base + belongs_to :user + + validates :user_id, :key, :value, presence: true + validates :key, uniqueness: { scope: [:user_id] } +end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index f2315bb3dbb..5f710961f95 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -50,7 +50,7 @@ class WikiPage # The Gitlab ProjectWiki instance. attr_reader :wiki - # The raw Gollum::Page instance. + # The raw Gitlab::Git::WikiPage instance. attr_reader :page # The attributes Hash used for storing and validating @@ -75,7 +75,7 @@ class WikiPage if @attributes[:slug].present? @attributes[:slug] else - wiki.wiki.preview_page(title, '', format).url_path + wiki.wiki.preview_slug(title, format) end end @@ -131,7 +131,7 @@ class WikiPage def versions return [] unless persisted? - @page.versions + wiki.wiki.page_versions(@page.path) end def commit @@ -264,8 +264,8 @@ class WikiPage end page_title, page_dir = wiki.page_title_and_dir(page_details) - gollum_wiki = wiki.wiki - @page = gollum_wiki.paged(page_title, page_dir) + gitlab_git_wiki = wiki.wiki + @page = gitlab_git_wiki.page(title: page_title, dir: page_dir) set_attributes @persisted = errors.blank? |