diff options
Diffstat (limited to 'app/models')
55 files changed, 868 insertions, 412 deletions
diff --git a/app/models/appearance.rb b/app/models/appearance.rb index ff15689ecac..76cfe28742a 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -2,9 +2,8 @@ class Appearance < ActiveRecord::Base include CacheMarkdownField cache_markdown_field :description + cache_markdown_field :new_project_guidelines - validates :title, presence: true - validates :description, presence: true validates :logo, file_size: { maximum: 1.megabyte } validates :header_logo, file_size: { maximum: 1.megabyte } diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 5e16badabec..3117c98c846 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -172,6 +172,27 @@ class ApplicationSetting < ActiveRecord::Base end end + validates :gitaly_timeout_default, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + validates :gitaly_timeout_medium, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :gitaly_timeout_medium, + numericality: { less_than_or_equal_to: :gitaly_timeout_default }, + if: :gitaly_timeout_default + validates :gitaly_timeout_medium, + numericality: { greater_than_or_equal_to: :gitaly_timeout_fast }, + if: :gitaly_timeout_fast + + validates :gitaly_timeout_fast, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :gitaly_timeout_fast, + numericality: { less_than_or_equal_to: :gitaly_timeout_default }, + if: :gitaly_timeout_default + SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end @@ -276,7 +297,8 @@ class ApplicationSetting < ActiveRecord::Base koding_url: nil, max_artifacts_size: Settings.artifacts['max_size'], max_attachment_size: Settings.gitlab['max_attachment_size'], - password_authentication_enabled: Settings.gitlab['password_authentication_enabled'], + password_authentication_enabled_for_web: Settings.gitlab['signin_enabled'], + password_authentication_enabled_for_git: true, performance_bar_allowed_group_id: nil, rsa_key_restriction: 0, plantuml_enabled: false, @@ -295,10 +317,22 @@ class ApplicationSetting < ActiveRecord::Base sign_in_text: nil, signup_enabled: Settings.gitlab['signup_enabled'], terminal_max_session_time: 0, + throttle_unauthenticated_enabled: false, + throttle_unauthenticated_requests_per_period: 3600, + throttle_unauthenticated_period_in_seconds: 3600, + throttle_authenticated_web_enabled: false, + throttle_authenticated_web_requests_per_period: 7200, + throttle_authenticated_web_period_in_seconds: 3600, + throttle_authenticated_api_enabled: false, + throttle_authenticated_api_requests_per_period: 7200, + throttle_authenticated_api_period_in_seconds: 3600, two_factor_grace_period: 48, user_default_external: false, polling_interval_multiplier: 1, - usage_ping_enabled: Settings.gitlab['usage_ping_enabled'] + usage_ping_enabled: Settings.gitlab['usage_ping_enabled'], + gitaly_timeout_fast: 10, + gitaly_timeout_medium: 30, + gitaly_timeout_default: 55 } end @@ -465,6 +499,14 @@ class ApplicationSetting < ActiveRecord::Base has_attribute?(attr_name) ? public_send(attr_name) : FORBIDDEN_KEY_VALUE # rubocop:disable GitlabSecurity/PublicSend end + def allow_signup? + signup_enabled? && password_authentication_enabled_for_web? + end + + def password_authentication_enabled? + password_authentication_enabled_for_web? || password_authentication_enabled_for_git? + end + private def ensure_uuid! diff --git a/app/models/blob.rb b/app/models/blob.rb index ad0bc2e2ead..29e762724e3 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -76,12 +76,24 @@ class Blob < SimpleDelegator new(blob, project) end + def self.lazy(project, commit_id, path) + BatchLoader.for(commit_id: commit_id, path: path).batch do |items, loader| + project.repository.blobs_at(items.map(&:values)).each do |blob| + loader.call({ commit_id: blob.commit_id, path: blob.path }, blob) if blob + end + end + end + def initialize(blob, project = nil) @project = project super(blob) end + def inspect + "#<#{self.class.name} oid:#{id[0..8]} commit:#{commit_id[0..8]} path:#{path}>" + end + # Returns the data of the blob. # # If the blob is a text based blob the content is converted to UTF-8 and any @@ -95,7 +107,10 @@ class Blob < SimpleDelegator end def load_all_data! - super(project.repository) if project + # Endpoint needed: gitlab-org/gitaly#756 + Gitlab::GitalyClient.allow_n_plus_1_calls do + super(project.repository) if project + end end def no_highlighting? diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1b2b0d17910..d2402b55184 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -1,5 +1,6 @@ module Ci class Build < CommitStatus + prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue include Presentable @@ -10,9 +11,14 @@ module Ci belongs_to :erased_by, class_name: 'User' has_many :deployments, as: :deployable + has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @persisted_environment ||= Environment.find_by( @@ -31,15 +37,37 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } - scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) } + scope :with_artifacts, ->() do + where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', + '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id')) + end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } - mount_uploader :artifacts_file, ArtifactUploader - mount_uploader :artifacts_metadata, ArtifactUploader + scope :matches_tag_ids, -> (tag_ids) do + matcher = ::ActsAsTaggableOn::Tagging + .where(taggable_type: CommitStatus) + .where(context: 'tags') + .where('taggable_id = ci_builds.id') + .where.not(tag_id: tag_ids).select('1') + + where("NOT EXISTS (?)", matcher) + end + + scope :with_any_tags, -> do + matcher = ::ActsAsTaggableOn::Tagging + .where(taggable_type: CommitStatus) + .where(context: 'tags') + .where('taggable_id = ci_builds.id').select('1') + + where("EXISTS (?)", matcher) + end + + mount_uploader :legacy_artifacts_file, LegacyArtifactUploader, mount_on: :artifacts_file + mount_uploader :legacy_artifacts_metadata, LegacyArtifactUploader, mount_on: :artifacts_metadata acts_as_taggable @@ -104,6 +132,7 @@ module Ci end before_transition any => [:failed] do |build| + next unless build.project next if build.retries_max.zero? if build.retries_count < build.retries_max @@ -243,7 +272,7 @@ module Ci @merge_request ||= begin - merge_requests = MergeRequest.includes(:merge_request_diff) + merge_requests = MergeRequest.includes(:latest_merge_request_diff) .where(source_branch: ref, source_project: pipeline.project) .reorder(iid: :desc) @@ -317,6 +346,7 @@ module Ci def execute_hooks return unless project + build_data = Gitlab::DataBuilder::Build.build(self) project.execute_hooks(build_data.dup, :job_hooks) project.execute_services(build_data.dup, :job_hooks) @@ -324,14 +354,6 @@ module Ci project.running_or_pending_build_count(force: true) end - def artifacts? - !artifacts_expired? && artifacts_file.exists? - end - - def artifacts_metadata? - artifacts? && artifacts_metadata.exists? - end - def artifacts_metadata_entry(path, **options) metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( artifacts_metadata.path, @@ -384,6 +406,7 @@ module Ci def keep_artifacts! self.update(artifacts_expire_at: nil) + self.job_artifacts.update_all(expire_at: nil) end def coverage_regex @@ -471,11 +494,7 @@ module Ci private def update_artifacts_size - self.artifacts_size = if artifacts_file.exists? - artifacts_file.size - else - nil - end + self.artifacts_size = legacy_artifacts_file&.size end def erase_trace! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb new file mode 100644 index 00000000000..84fc6863567 --- /dev/null +++ b/app/models/ci/job_artifact.rb @@ -0,0 +1,36 @@ +module Ci + class JobArtifact < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :project + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + + before_save :set_size, if: :file_changed? + + mount_uploader :file, JobArtifactUploader + + enum file_type: { + archive: 1, + metadata: 2 + } + + def self.artifacts_size_for(project) + self.where(project: project).sum(:size) + end + + def set_size + self.size = file.size + end + + def expire_in + expire_at - Time.now if expire_at + end + + def expire_in=(value) + self.expire_at = + if value + ChronicDuration.parse(value)&.seconds&.from_now + end + end + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 19814864e50..eebbf7c4218 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -40,7 +40,6 @@ module Ci validates :status, presence: { unless: :importing? } validate :valid_commit_sha, unless: :importing? - after_initialize :set_config_source, if: :new_record? after_create :keep_around_commits, unless: :importing? enum source: { @@ -149,34 +148,70 @@ module Ci end end - # ref can't be HEAD or SHA, can only be branch/tag name - scope :latest, ->(ref = nil) do - max_id = unscope(:select) - .select("max(#{quoted_table_name}.id)") - .group(:ref, :sha) + scope :internal, -> { where(source: internal_sources) } - if ref - where(ref: ref, id: max_id.where(ref: ref)) - else - where(id: max_id) - end + # Returns the pipelines in descending order (= newest first), optionally + # limited to a number of references. + # + # ref - The name (or names) of the branch(es)/tag(s) to limit the list of + # pipelines to. + def self.newest_first(ref = nil) + relation = order(id: :desc) + + ref ? relation.where(ref: ref) : relation end - scope :internal, -> { where(source: internal_sources) } def self.latest_status(ref = nil) - latest(ref).status + newest_first(ref).pluck(:status).first end def self.latest_successful_for(ref) - success.latest(ref).order(id: :desc).first + newest_first(ref).success.take end def self.latest_successful_for_refs(refs) - success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash| + relation = newest_first(refs).success + + relation.each_with_object({}) do |pipeline, hash| hash[pipeline.ref] ||= pipeline end end + # Returns a Hash containing the latest pipeline status for every given + # commit. + # + # The keys of this Hash are the commit SHAs, the values the statuses. + # + # commits - The list of commit SHAs to get the status for. + # ref - The ref to scope the data to (e.g. "master"). If the ref is not + # given we simply get the latest status for the commits, regardless + # of what refs their pipelines belong to. + def self.latest_status_per_commit(commits, ref = nil) + p1 = arel_table + p2 = arel_table.alias + + # This LEFT JOIN will filter out all but the newest row for every + # combination of (project_id, sha) or (project_id, sha, ref) if a ref is + # given. + cond = p1[:sha].eq(p2[:sha]) + .and(p1[:project_id].eq(p2[:project_id])) + .and(p1[:id].lt(p2[:id])) + + cond = cond.and(p1[:ref].eq(p2[:ref])) if ref + join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond) + + relation = select(:sha, :status) + .where(sha: commits) + .where(p2[:id].eq(nil)) + .joins(join.join_sources) + + relation = relation.where(ref: ref) if ref + + relation.each_with_object({}) do |row, hash| + hash[row[:sha]] = row[:status] + end + end + def self.truncate_sha(sha) sha[0...8] end @@ -300,8 +335,10 @@ module Ci def latest? return false unless ref + commit = project.commit(ref) return false unless commit + commit.sha == sha end @@ -327,7 +364,7 @@ module Ci end def has_kubernetes_active? - project.kubernetes_service&.active? + project.deployment_platform&.active? end def has_stage_seeds? @@ -469,7 +506,10 @@ module Ci end def latest_builds_with_artifacts - @latest_builds_with_artifacts ||= builds.latest.with_artifacts + # We purposely cast the builds to an Array here. Because we always use the + # rows if there are more than 0 this prevents us from having to run two + # queries: one to get the count and one to get the rows. + @latest_builds_with_artifacts ||= builds.latest.with_artifacts.to_a end private diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index c6509f89117..dcbb397fb78 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -1,6 +1,7 @@ module Ci class Runner < ActiveRecord::Base extend Gitlab::Ci::Model + include Gitlab::SQL::Pattern RUNNER_QUEUE_EXPIRY_TIME = 60.minutes ONLINE_CONTACT_TIMEOUT = 1.hour @@ -59,10 +60,7 @@ module Ci # # Returns an ActiveRecord::Relation. def self.search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:token].matches(pattern).or(t[:description].matches(pattern))) + fuzzy_search(query, [:token, :description]) end def self.contact_time_deadline @@ -114,7 +112,7 @@ module Ci def can_pick?(build) return false if self.ref_protected? && !build.protected? - assignable_for?(build.project) && accepting_tags?(build) + assignable_for?(build.project_id) && accepting_tags?(build) end def only_for?(project) @@ -173,8 +171,8 @@ module Ci end end - def assignable_for?(project) - is_shared? || projects.exists?(id: project.id) + def assignable_for?(project_id) + is_shared? || projects.exists?(id: project_id) end def accepting_tags?(build) diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 185d9473aab..55419189282 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -17,8 +17,7 @@ module Clusters # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true - # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration - has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true has_one :application_helm, class_name: 'Clusters::Applications::Helm' has_one :application_ingress, class_name: 'Clusters::Applications::Ingress' @@ -29,15 +28,9 @@ module Clusters validates :name, cluster_name: true validate :restrict_modification, on: :update - # TODO: Move back this into Clusters::Platforms::Kubernetes in 10.3 - # We need callback here because `enabled` belongs to Clusters::Cluster - # Callbacks in Clusters::Platforms::Kubernetes will not be called after update - after_save :update_kubernetes_integration! - delegate :status, to: :provider, allow_nil: true delegate :status_reason, to: :provider, allow_nil: true delegate :on_creation?, to: :provider, allow_nil: true - delegate :update_kubernetes_integration!, to: :platform, allow_nil: true delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true delegate :installed?, to: :application_helm, prefix: true, allow_nil: true @@ -62,6 +55,10 @@ module Clusters end end + def created? + status_name == :created + end + def applications [ application_helm || build_application_helm, @@ -77,6 +74,10 @@ module Clusters return platform_kubernetes if kubernetes? end + def managed? + !user? + end + def first_project return @first_project if defined?(@first_project) diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 6dc1ee810d3..9160a169452 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -1,7 +1,12 @@ module Clusters module Platforms class Kubernetes < ActiveRecord::Base + include Gitlab::CurrentSettings + include Gitlab::Kubernetes + include ReactiveCaching + self.table_name = 'cluster_platforms_kubernetes' + self.reactive_cache_key = ->(kubernetes) { [kubernetes.class.model_name.singular, kubernetes.id] } belongs_to :cluster, inverse_of: :platform_kubernetes, class_name: 'Clusters::Cluster' @@ -29,19 +34,17 @@ module Clusters validates :api_url, url: true, presence: true validates :token, presence: true - # TODO: Glue code till we migrate Kubernetes Integration into Platforms::Kubernetes - after_destroy :destroy_kubernetes_integration! + validate :prevent_modification, on: :update + + after_save :clear_reactive_cache! alias_attribute :ca_pem, :ca_cert delegate :project, to: :cluster, allow_nil: true delegate :enabled?, to: :cluster, allow_nil: true + delegate :managed?, to: :cluster, allow_nil: true - class << self - def namespace_for_project(project) - "#{project.path}-#{project.id}" - end - end + alias_method :active?, :enabled? def actual_namespace if namespace.present? @@ -51,58 +54,138 @@ module Clusters end end - def default_namespace - self.class.namespace_for_project(project) if project + def predefined_variables + config = YAML.dump(kubeconfig) + + variables = [ + { key: 'KUBE_URL', value: api_url, public: true }, + { key: 'KUBE_TOKEN', value: token, public: false }, + { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true }, + { key: 'KUBECONFIG', value: config, public: false, file: true } + ] + + if ca_pem.present? + variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true } + variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true } + end + + variables end - def kubeclient - @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service? + # Constructs a list of terminals from the reactive cache + # + # Returns nil if the cache is empty, in which case you should try again a + # short time later + def terminals(environment) + with_reactive_cache do |data| + pods = filter_by_label(data[:pods], app: environment.slug) + terminals = pods.flat_map { |pod| terminals_for_pod(api_url, actual_namespace, pod) } + terminals.each { |terminal| add_terminal_auth(terminal, terminal_auth) } + end end - def update_kubernetes_integration! - raise 'Kubernetes service already configured' unless manages_kubernetes_service? + # Caches resources in the namespace so other calls don't need to block on + # network access + def calculate_reactive_cache + return unless enabled? && project && !project.pending_delete? - # This is neccesary, otheriwse enabled? returns true even though cluster updated with enabled: false - cluster.reload + # We may want to cache extra things in the future + { pods: read_pods } + end + + def kubeclient + @kubeclient ||= build_kubeclient! + end + + private - ensure_kubernetes_service&.update!( - active: enabled?, - api_url: api_url, - namespace: namespace, + def kubeconfig + to_kubeconfig( + url: api_url, + namespace: actual_namespace, token: token, - ca_pem: ca_cert - ) + ca_pem: ca_pem) end - def active? - manages_kubernetes_service? + def default_namespace + return unless project + + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') end - private + def build_kubeclient!(api_path: 'api', api_version: 'v1') + raise "Incomplete settings" unless api_url && actual_namespace - def enforce_namespace_to_lower_case - self.namespace = self.namespace&.downcase + unless (username && password) || token + raise "Either username/password or token is required to access API" + end + + ::Kubeclient::Client.new( + join_api_url(api_path), + api_version, + auth_options: kubeclient_auth_options, + ssl_options: kubeclient_ssl_options, + http_proxy_uri: ENV['http_proxy'] + ) end - # TODO: glue code till we migrate Kubernetes Service into Platforms::Kubernetes class - def manages_kubernetes_service? - return true unless kubernetes_service&.active? + # Returns a hash of all pods in the namespace + def read_pods + kubeclient = build_kubeclient! + + kubeclient.get_pods(namespace: actual_namespace).as_json + rescue KubeException => err + raise err unless err.error_code == 404 + + [] + end + + def kubeclient_ssl_options + opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER } + + if ca_pem.present? + opts[:cert_store] = OpenSSL::X509::Store.new + opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem)) + end + + opts + end - kubernetes_service.api_url == api_url + def kubeclient_auth_options + { bearer_token: token } end - def destroy_kubernetes_integration! - return unless manages_kubernetes_service? + def join_api_url(api_path) + url = URI.parse(api_url) + prefix = url.path.sub(%r{/+\z}, '') + + url.path = [prefix, api_path].join("/") - kubernetes_service&.destroy! + url.to_s end - def kubernetes_service - @kubernetes_service ||= project&.kubernetes_service + def terminal_auth + { + token: token, + ca_pem: ca_pem, + max_session_time: current_application_settings.terminal_max_session_time + } end - def ensure_kubernetes_service - @kubernetes_service ||= kubernetes_service || project&.build_kubernetes_service + def enforce_namespace_to_lower_case + self.namespace = self.namespace&.downcase + end + + def prevent_modification + return unless managed? + + if api_url_changed? || token_changed? || ca_pem_changed? + errors.add(:base, "cannot modify managed cluster") + return false + end + + true end end end diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index ee2e43ee9dd..7fac32466ab 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -56,6 +56,7 @@ module Clusters before_transition any => [:creating] do |provider, transition| operation_id = transition.args.first raise ArgumentError.new('operation_id is required') unless operation_id.present? + provider.operation_id = operation_id end diff --git a/app/models/commit.rb b/app/models/commit.rb index 6dba154a6ea..6b28d290f99 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -80,10 +80,11 @@ class Commit @raw = raw_commit @project = project + @statuses = {} end def id - @raw.id + raw.id end def ==(other) @@ -108,12 +109,12 @@ class Commit @link_reference_pattern ||= super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})/) end - def to_reference(from_project = nil, full: false) - commit_reference(from_project, id, full: full) + def to_reference(from = nil, full: false) + commit_reference(from, id, full: full) end - def reference_link_text(from_project = nil, full: false) - commit_reference(from_project, short_id, full: full) + def reference_link_text(from = nil, full: false) + commit_reference(from, short_id, full: full) end def diff_line_count @@ -236,11 +237,13 @@ class Commit end def status(ref = nil) - @statuses ||= {} - return @statuses[ref] if @statuses.key?(ref) - @statuses[ref] = pipelines.latest_status(ref) + @statuses[ref] = project.pipelines.latest_status_per_commit(id, ref)[id] + end + + def set_status_for_ref(ref, status) + @statuses[ref] = status end def signature @@ -358,7 +361,7 @@ class Commit @deltas ||= raw.deltas end - def diffs(diff_options = nil) + def diffs(diff_options = {}) Gitlab::Diff::FileCollection::Commit.new(self, diff_options: diff_options) end @@ -378,8 +381,8 @@ class Commit private - def commit_reference(from_project, referable_commit_id, full: false) - reference = project.to_reference(from_project, full: full) + def commit_reference(from, referable_commit_id, full: false) + reference = project.to_reference(from, full: full) if reference.present? "#{reference}#{self.class.reference_prefix}#{referable_commit_id}" diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb new file mode 100644 index 00000000000..dd93af9df64 --- /dev/null +++ b/app/models/commit_collection.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# A collection of Commit instances for a specific project and Git reference. +class CommitCollection + include Enumerable + + attr_reader :project, :ref, :commits + + # project - The project the commits belong to. + # commits - The Commit instances to store. + # ref - The name of the ref (e.g. "master"). + def initialize(project, commits, ref = nil) + @project = project + @commits = commits + @ref = ref + end + + def each(&block) + commits.each(&block) + end + + # Sets the pipeline status for every commit. + # + # Setting this status ahead of time removes the need for running a query for + # every commit we're displaying. + def with_pipeline_status + statuses = project.pipelines.latest_status_per_commit(map(&:id), ref) + + each do |commit| + commit.set_status_for_ref(ref, statuses[commit.id]) + end + + self + end + + def respond_to_missing?(message, inc_private = false) + commits.respond_to?(message, inc_private) + end + + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(message, *args, &block) + commits.public_send(message, *args, &block) + end +end diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 84e2e8a5dd5..b93c111dabc 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -89,8 +89,8 @@ class CommitRange alias_method :id, :to_s - def to_reference(from_project = nil, full: false) - project_reference = project.to_reference(from_project, full: full) + def to_reference(from = nil, full: false) + project_reference = project.to_reference(from, full: full) if project_reference.present? project_reference + self.class.reference_prefix + self.id @@ -99,8 +99,8 @@ class CommitRange end end - def reference_link_text(from_project = nil) - project_reference = project.to_reference(from_project) + def reference_link_text(from = nil) + project_reference = project.to_reference(from) reference = ref_from + notation + ref_to if project_reference.present? diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 6b07dbdf3ea..ee21ed8e420 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -17,6 +17,7 @@ class CommitStatus < ActiveRecord::Base validates :name, presence: true, unless: :importing? alias_attribute :author, :user + alias_attribute :pipeline_id, :commit_id scope :failed_but_allowed, -> do where(allow_failure: true, status: [:failed, :canceled]) @@ -103,26 +104,29 @@ class CommitStatus < ActiveRecord::Base end after_transition do |commit_status, transition| + next unless commit_status.project next if transition.loopback? commit_status.run_after_commit do - if pipeline + if pipeline_id if complete? || manual? - PipelineProcessWorker.perform_async(pipeline.id) + PipelineProcessWorker.perform_async(pipeline_id) else - PipelineUpdateWorker.perform_async(pipeline.id) + PipelineUpdateWorker.perform_async(pipeline_id) end end - StageUpdateWorker.perform_async(commit_status.stage_id) - ExpireJobCacheWorker.perform_async(commit_status.id) + StageUpdateWorker.perform_async(stage_id) + ExpireJobCacheWorker.perform_async(id) end end after_transition any => :failed do |commit_status| + next unless commit_status.project + commit_status.run_after_commit do MergeRequests::AddTodoWhenBuildFailsService - .new(pipeline.project, nil).execute(self) + .new(project, nil).execute(self) end end end diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb new file mode 100644 index 00000000000..0460439e9e6 --- /dev/null +++ b/app/models/concerns/artifact_migratable.rb @@ -0,0 +1,45 @@ +# Adapter class to unify the interface between mounted uploaders and the +# Ci::Artifact model +# Meant to be prepended so the interface can stay the same +module ArtifactMigratable + def artifacts_file + job_artifacts_archive&.file || legacy_artifacts_file + end + + def artifacts_metadata + job_artifacts_metadata&.file || legacy_artifacts_metadata + end + + def artifacts? + !artifacts_expired? && artifacts_file.exists? + end + + def artifacts_metadata? + artifacts? && artifacts_metadata.exists? + end + + def artifacts_file_changed? + job_artifacts_archive&.file_changed? || attribute_changed?(:artifacts_file) + end + + def remove_artifacts_file! + if job_artifacts_archive + job_artifacts_archive.destroy + else + remove_legacy_artifacts_file! + end + end + + def remove_artifacts_metadata! + if job_artifacts_metadata + job_artifacts_metadata.destroy + else + remove_legacy_artifacts_metadata! + end + end + + def artifacts_size + read_attribute(:artifacts_size).to_i + + job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i + end +end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index 9adc309a22b..d8394415362 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -98,6 +98,7 @@ module Awardable def create_award_emoji(name, current_user) return unless emoji_awardable? + award_emoji.create(name: normalize_name(name), user: current_user) end diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb index 9585b5583dc..8a241e4374a 100644 --- a/app/models/concerns/has_variable.rb +++ b/app/models/concerns/has_variable.rb @@ -16,6 +16,10 @@ module HasVariable key: Gitlab::Application.secrets.db_key_base, algorithm: 'aes-256-cbc' + def key=(new_key) + super(new_key.to_s.strip) + end + def to_runner_variable { key: key, value: value, public: false } end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 35090181bd9..5ca4a7086cb 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -122,9 +122,7 @@ module Issuable # # Returns an ActiveRecord::Relation. def search(query) - title = to_fuzzy_arel(:title, query) - - where(title) + fuzzy_search(query, [:title]) end # Searches for records with a matching title or description. @@ -135,10 +133,7 @@ module Issuable # # Returns an ActiveRecord::Relation. def full_search(query) - title = to_fuzzy_arel(:title, query) - description = to_fuzzy_arel(:description, query) - - where(title&.or(description)) + fuzzy_search(query, [:title, :description]) end def sort(method, excluded_labels: []) @@ -255,8 +250,10 @@ module Issuable participants(user).include?(user) end - def to_hook_data(user, old_labels: [], old_assignees: [], old_total_time_spent: nil) + def to_hook_data(user, old_associations: {}) changes = previous_changes + old_labels = old_associations.fetch(:labels, []) + old_assignees = old_associations.fetch(:assignees, []) if old_labels != labels changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)] @@ -270,8 +267,12 @@ module Issuable end end - if old_total_time_spent != total_time_spent - changes[:total_time_spent] = [old_total_time_spent, total_time_spent] + if self.respond_to?(:total_time_spent) + old_total_time_spent = old_associations.fetch(:total_time_spent, nil) + + if old_total_time_spent != total_time_spent + changes[:total_time_spent] = [old_total_time_spent, total_time_spent] + end end Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes) @@ -345,4 +346,11 @@ module Issuable def first_contribution? false end + + ## + # Overriden in MergeRequest + # + def wipless_title_changed(old_title) + old_title != title + end end diff --git a/app/models/concerns/manual_inverse_association.rb b/app/models/concerns/manual_inverse_association.rb new file mode 100644 index 00000000000..0fca8feaf89 --- /dev/null +++ b/app/models/concerns/manual_inverse_association.rb @@ -0,0 +1,17 @@ +module ManualInverseAssociation + extend ActiveSupport::Concern + + module ClassMethods + def manual_inverse_association(association, inverse) + define_method(association) do |*args| + super(*args).tap do |value| + next unless value + + child_association = value.association(inverse) + child_association.set_inverse_instance(self) + child_association.target = self + end + end + end + end +end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index 1db6b2d2fa2..b43eaeaeea0 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -31,11 +31,11 @@ module Mentionable # # By default this will be the class name and the result of calling # `to_reference` on the object. - def gfm_reference(from_project = nil) + def gfm_reference(from = nil) # "MergeRequest" > "merge_request" > "Merge request" > "merge request" friendly_name = self.class.to_s.underscore.humanize.downcase - "#{friendly_name} #{to_reference(from_project)}" + "#{friendly_name} #{to_reference(from)}" end # The GFM reference to this Mentionable, which shouldn't be included in its #references. diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb index fde1cc44afa..e62f42e8e70 100644 --- a/app/models/concerns/protected_branch_access.rb +++ b/app/models/concerns/protected_branch_access.rb @@ -1,12 +1,6 @@ module ProtectedBranchAccess extend ActiveSupport::Concern - ALLOWED_ACCESS_LEVELS ||= [ - Gitlab::Access::MASTER, - Gitlab::Access::DEVELOPER, - Gitlab::Access::NO_ACCESS - ].freeze - included do include ProtectedRefAccess @@ -14,18 +8,6 @@ module ProtectedBranchAccess delegate :project, to: :protected_branch - validates :access_level, presence: true, inclusion: { - in: ALLOWED_ACCESS_LEVELS - } - - def self.human_access_levels - { - Gitlab::Access::MASTER => "Masters", - Gitlab::Access::DEVELOPER => "Developers + Masters", - Gitlab::Access::NO_ACCESS => "No one" - }.with_indifferent_access - end - def check_access(user) return false if access_level == Gitlab::Access::NO_ACCESS diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index c4f158e569a..80c9f7d4eb4 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -1,13 +1,35 @@ module ProtectedRefAccess extend ActiveSupport::Concern + ALLOWED_ACCESS_LEVELS = [ + Gitlab::Access::MASTER, + Gitlab::Access::DEVELOPER, + Gitlab::Access::NO_ACCESS + ].freeze + + HUMAN_ACCESS_LEVELS = { + Gitlab::Access::MASTER => "Masters".freeze, + Gitlab::Access::DEVELOPER => "Developers + Masters".freeze, + Gitlab::Access::NO_ACCESS => "No one".freeze + }.freeze + included do scope :master, -> { where(access_level: Gitlab::Access::MASTER) } scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } + + validates :access_level, presence: true, if: :role?, inclusion: { + in: ALLOWED_ACCESS_LEVELS + } end def humanize - self.class.human_access_levels[self.access_level] + HUMAN_ACCESS_LEVELS[self.access_level] + end + + # CE access levels are always role-based, + # where as EE allows groups and users too + def role? + true end def check_access(user) diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb index 78ac4f324e7..b782e85717e 100644 --- a/app/models/concerns/referable.rb +++ b/app/models/concerns/referable.rb @@ -7,7 +7,7 @@ module Referable # Returns the String necessary to reference this object in Markdown # - # from_project - Refering Project object + # from - Referring parent object # # This should be overridden by the including class. # @@ -17,12 +17,12 @@ module Referable # Issue.last.to_reference(other_project) # => "cross-project#1" # # Returns a String - def to_reference(_from_project = nil, full:) + def to_reference(_from = nil, full:) '' end - def reference_link_text(from_project = nil) - to_reference(from_project) + def reference_link_text(from = nil) + to_reference(from) end included do diff --git a/app/models/email.rb b/app/models/email.rb index 2da8b050149..d6516761f0a 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -1,5 +1,6 @@ class Email < ActiveRecord::Base include Sortable + include Gitlab::SQL::Pattern belongs_to :user diff --git a/app/models/environment.rb b/app/models/environment.rb index 21a028e351c..bf69b4c50f0 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -138,11 +138,11 @@ class Environment < ActiveRecord::Base end def has_terminals? - project.deployment_service.present? && available? && last_deployment.present? + project.deployment_platform.present? && available? && last_deployment.present? end def terminals - project.deployment_service.terminals(self) if has_terminals? + project.deployment_platform.terminals(self) if has_terminals? end def has_metrics? diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb index 9ff56f229bc..2aaba2e4c90 100644 --- a/app/models/external_issue.rb +++ b/app/models/external_issue.rb @@ -38,11 +38,11 @@ class ExternalIssue @project.id end - def to_reference(_from_project = nil, full: nil) + def to_reference(_from = nil, full: nil) id end - def reference_link_text(from_project = nil) + def reference_link_text(from = nil) return "##{id}" if id =~ /^\d+$/ id diff --git a/app/models/fork_network_member.rb b/app/models/fork_network_member.rb index 6a9b52a1ef8..eb9417dc34f 100644 --- a/app/models/fork_network_member.rb +++ b/app/models/fork_network_member.rb @@ -4,4 +4,14 @@ class ForkNetworkMember < ActiveRecord::Base belongs_to :forked_from_project, class_name: 'Project' validates :fork_network, :project, presence: true + + after_destroy :cleanup_fork_network + + private + + def cleanup_fork_network + # Explicitly using `#count` makes sure we have the correct number if the + # relation was loaded in the fork_network. + fork_network.destroy if fork_network.fork_network_members.count == 0 + end end diff --git a/app/models/group.rb b/app/models/group.rb index 8cf632fb566..505e943e464 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Group < Namespace include Gitlab::ConfigHelper + include AfterCommitQueue include AccessRequestable include Avatarable include Referable @@ -50,20 +51,6 @@ class Group < Namespace Gitlab::Database.postgresql? end - # Searches for groups matching the given query. - # - # This method uses ILIKE on PostgreSQL and LIKE on MySQL. - # - # query - The search query as a String - # - # Returns an ActiveRecord::Relation. - def search(query) - table = Namespace.arel_table - pattern = "%#{query}%" - - where(table[:name].matches(pattern).or(table[:path].matches(pattern))) - end - def sort(method) if method == 'storage_size_desc' # storage_size is a virtual column so we need to @@ -97,7 +84,7 @@ class Group < Namespace end end - def to_reference(_from_project = nil, full: nil) + def to_reference(_from = nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end @@ -303,6 +290,14 @@ class Group < Namespace "#{parent.full_path}/#{path_was}" end + def group_member(user) + if group_members.loaded? + group_members.find { |gm| gm.user_id == user.id } + else + group_members.find_by(user_id: user) + end + end + private def update_two_factor_requirement diff --git a/app/models/identity.rb b/app/models/identity.rb index ac8094b610e..ff811e19f8a 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -1,18 +1,27 @@ class Identity < ActiveRecord::Base include Sortable include CaseSensitivity + belongs_to :user validates :provider, presence: true - validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider } + validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider, case_sensitive: false } validates :user_id, uniqueness: { scope: :provider } + scope :with_provider, ->(provider) { where(provider: provider) } scope :with_extern_uid, ->(provider, extern_uid) do - extern_uid = Gitlab::LDAP::Person.normalize_dn(extern_uid) if provider.starts_with?('ldap') - where(extern_uid: extern_uid, provider: provider) + iwhere(extern_uid: normalize_uid(provider, extern_uid)).with_provider(provider) end def ldap? provider.starts_with?('ldap') end + + def self.normalize_uid(provider, uid) + if provider.to_s.starts_with?('ldap') + Gitlab::LDAP::Person.normalize_dn(uid) + else + uid.to_s + end + end end diff --git a/app/models/issue.rb b/app/models/issue.rb index b5abc8f57b0..d6ef58d150b 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -49,7 +49,6 @@ class Issue < ActiveRecord::Base scope :public_only, -> { where(confidential: false) } after_save :expire_etag_cache - after_commit :update_project_counter_caches, on: :destroy attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true @@ -246,7 +245,12 @@ class Issue < ActiveRecord::Base def as_json(options = {}) super(options).tap do |json| - json[:subscribed] = subscribed?(options[:user], project) if options.key?(:user) && options[:user] + if options.key?(:sidebar_endpoints) && project + url_helper = Gitlab::Routing.url_helpers + + json.merge!(issue_sidebar_endpoint: url_helper.project_issue_path(project, self, format: :json, serializer: 'sidebar'), + toggle_subscription_endpoint: url_helper.toggle_subscription_project_issue_path(project, self)) + end if options.key?(:labels) json[:labels] = labels.as_json( diff --git a/app/models/key.rb b/app/models/key.rb index f119b15c737..a3f8a5d6dc7 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -2,6 +2,7 @@ require 'digest/md5' class Key < ActiveRecord::Base include Gitlab::CurrentSettings + include AfterCommitQueue include Sortable belongs_to :user @@ -27,8 +28,10 @@ class Key < ActiveRecord::Base after_commit :add_to_shell, on: :create after_create :post_create_hook + after_create :refresh_user_cache after_commit :remove_from_shell, on: :destroy after_destroy :post_destroy_hook + after_destroy :refresh_user_cache def key=(value) value&.delete!("\n\r") @@ -76,6 +79,12 @@ class Key < ActiveRecord::Base ) end + def refresh_user_cache + return unless user + + Users::KeysCountService.new(user).refresh_cache + end + def post_destroy_hook SystemHooksService.new.execute_hooks_for(self, :destroy) end diff --git a/app/models/label.rb b/app/models/label.rb index 899028a01a0..b5bfa6ea2dd 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -165,12 +165,12 @@ class Label < ActiveRecord::Base # # Returns a String # - def to_reference(from_project = nil, target_project: nil, format: :id, full: false) + def to_reference(from = nil, target_project: nil, format: :id, full: false) format_reference = label_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" - if from_project - "#{from_project.to_reference(target_project, full: full)}#{reference}" + if from + "#{from.to_reference(target_project, full: full)}#{reference}" else reference end diff --git a/app/models/member.rb b/app/models/member.rb index cbbd58f2eaf..2fe5fda985f 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -1,4 +1,5 @@ class Member < ActiveRecord::Base + include AfterCommitQueue include Sortable include Importable include Expirable diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index f1a5cc73e83..bbc01e9677c 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -5,6 +5,8 @@ class MergeRequest < ActiveRecord::Base include Referable include IgnorableColumn include TimeTrackable + include ManualInverseAssociation + include EachBatch ignore_column :locked_at, :ref_fetched @@ -14,9 +16,28 @@ class MergeRequest < ActiveRecord::Base belongs_to :merge_user, class_name: "User" has_many :merge_request_diffs + has_one :merge_request_diff, -> { order('merge_request_diffs.id DESC') }, inverse_of: :merge_request + belongs_to :latest_merge_request_diff, class_name: 'MergeRequestDiff' + manual_inverse_association :latest_merge_request_diff, :merge_request + + # This is the same as latest_merge_request_diff unless: + # 1. There are arguments - in which case we might be trying to force-reload. + # 2. This association is already loaded. + # 3. The latest diff does not exist. + # + # The second one in particular is important - MergeRequestDiff#merge_request + # is the inverse of MergeRequest#merge_request_diff, which means it may not be + # the latest diff, because we could have loaded any diff from this particular + # MR. If we haven't already loaded a diff, then it's fine to load the latest. + def merge_request_diff(*args) + fallback = latest_merge_request_diff if args.empty? && !association(:merge_request_diff).loaded? + + fallback || super + end + belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline" has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -31,7 +52,6 @@ class MergeRequest < ActiveRecord::Base after_create :ensure_merge_request_diff, unless: :importing? after_update :reload_diff_if_branch_changed - after_commit :update_project_counter_caches, on: :destroy # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests @@ -167,6 +187,22 @@ class MergeRequest < ActiveRecord::Base where("merge_requests.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end + # This is used after project import, to reset the IDs to the correct + # values. It is not intended to be called without having already scoped the + # relation. + def self.set_latest_merge_request_diff_ids! + update = ' + latest_merge_request_diff_id = ( + SELECT MAX(id) + FROM merge_request_diffs + WHERE merge_requests.id = merge_request_diffs.merge_request_id + )'.squish + + self.each_batch do |batch| + batch.update_all(update) + end + end + WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze def self.work_in_progress?(title) @@ -181,6 +217,12 @@ class MergeRequest < ActiveRecord::Base work_in_progress?(title) ? title : "WIP: #{title}" end + # Verifies if title has changed not taking into account WIP prefix + # for merge requests. + def wipless_title_changed(old_title) + self.class.wipless_title(old_title) != self.wipless_title + end + def hook_attrs Gitlab::HookData::MergeRequestBuilder.new(self).build end @@ -241,9 +283,9 @@ class MergeRequest < ActiveRecord::Base if persisted? merge_request_diff.commit_shas elsif compare_commits - compare_commits.reverse.map(&:sha) + compare_commits.to_a.reverse.map(&:sha) else - [] + Array(diff_head_sha) end end @@ -322,16 +364,28 @@ class MergeRequest < ActiveRecord::Base # We use these attributes to force these to the intended values. attr_writer :target_branch_sha, :source_branch_sha + def source_branch_ref + return @source_branch_sha if @source_branch_sha + return unless source_branch + + Gitlab::Git::BRANCH_REF_PREFIX + source_branch + end + + def target_branch_ref + return @target_branch_sha if @target_branch_sha + return unless target_branch + + Gitlab::Git::BRANCH_REF_PREFIX + target_branch + end + def source_branch_head return unless source_project - source_branch_ref = @source_branch_sha || source_branch source_project.repository.commit(source_branch_ref) if source_branch_ref end def target_branch_head - target_branch_ref = @target_branch_sha || target_branch - target_project.repository.commit(target_branch_ref) if target_branch_ref + target_project.repository.commit(target_branch_ref) end def branch_merge_base_commit @@ -440,7 +494,7 @@ class MergeRequest < ActiveRecord::Base def merge_request_diff_for(diff_refs_or_sha) @merge_request_diffs_by_diff_refs_or_sha ||= Hash.new do |h, diff_refs_or_sha| - diffs = merge_request_diffs.viewable.select_without_diff + diffs = merge_request_diffs.viewable h[diff_refs_or_sha] = if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs) diffs.find_by_diff_refs(diff_refs_or_sha) @@ -845,7 +899,8 @@ class MergeRequest < ActiveRecord::Base def compute_diverged_commits_count return 0 unless source_branch_sha && target_branch_sha - Gitlab::Git::Commit.between(target_project.repository.raw_repository, source_branch_sha, target_branch_sha).size + target_project.repository + .count_commits_between(source_branch_sha, target_branch_sha) end private :compute_diverged_commits_count @@ -864,28 +919,18 @@ class MergeRequest < ActiveRecord::Base # Note that this could also return SHA from now dangling commits # def all_commit_shas - if persisted? - # MySQL doesn't support LIMIT in a subquery. - diffs_relation = - if Gitlab::Database.postgresql? - merge_request_diffs.order(id: :desc).limit(100) - else - merge_request_diffs - end + return commit_shas unless persisted? - column_shas = MergeRequestDiffCommit - .where(merge_request_diff: diffs_relation) - .limit(10_000) - .pluck('sha') + diffs_relation = merge_request_diffs - serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas) + # MySQL doesn't support LIMIT in a subquery. + diffs_relation = diffs_relation.recent if Gitlab::Database.postgresql? - (column_shas + serialised_shas).uniq - elsif compare_commits - compare_commits.to_a.reverse.map(&:id) - else - [diff_head_sha] - end + MergeRequestDiffCommit + .where(merge_request_diff: diffs_relation) + .limit(10_000) + .pluck('sha') + .uniq end def merge_commit diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 1eda0f9cbbd..c37aa0a594b 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -1,21 +1,21 @@ class MergeRequestDiff < ActiveRecord::Base include Sortable include Importable - include Gitlab::EncodingHelper + include ManualInverseAssociation + include IgnorableColumn - # Prevent store of diff if commits amount more then 500 + # Don't display more than 100 commits at once COMMITS_SAFE_SIZE = 100 - # Valid types of serialized diffs allowed by Gitlab::Git::Diff - VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta].freeze + ignore_column :st_commits, + :st_diffs belongs_to :merge_request + manual_inverse_association :merge_request, :merge_request_diff + has_many :merge_request_diff_files, -> { order(:merge_request_diff_id, :relative_order) } has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) } - serialize :st_commits # rubocop:disable Cop/ActiveRecordSerialize - serialize :st_diffs # rubocop:disable Cop/ActiveRecordSerialize - state_machine :state, initial: :empty do state :collected state :overflow @@ -29,6 +29,8 @@ class MergeRequestDiff < ActiveRecord::Base scope :viewable, -> { without_state(:empty) } + scope :recent, -> { order(id: :desc).limit(100) } + # All diff information is collected from repository after object is created. # It allows you to override variables like head_commit_sha before getting diff. after_create :save_git_content, unless: :importing? @@ -37,14 +39,6 @@ class MergeRequestDiff < ActiveRecord::Base find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha) end - def self.select_without_diff - select(column_names - ['st_diffs']) - end - - def st_commits - super || [] - end - # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content @@ -126,11 +120,7 @@ class MergeRequestDiff < ActiveRecord::Base end def commit_shas - if st_commits.present? - st_commits.map { |commit| commit[:id] } - else - merge_request_diff_commits.map(&:sha) - end + merge_request_diff_commits.map(&:sha) end def diff_refs=(new_diff_refs) @@ -194,7 +184,7 @@ class MergeRequestDiff < ActiveRecord::Base end def latest? - self == merge_request.merge_request_diff + self.id == merge_request.latest_merge_request_diff_id end def compare_with(sha) @@ -205,34 +195,11 @@ class MergeRequestDiff < ActiveRecord::Base end def commits_count - if st_commits.present? - st_commits.size - else - merge_request_diff_commits.size - end - end - - def utf8_st_diffs - return [] if st_diffs.blank? - - st_diffs.map do |diff| - diff.each do |k, v| - diff[k] = encode_utf8(v) if v.respond_to?(:encoding) - end - end + merge_request_diff_commits.size end private - # Old GitLab implementations may have generated diffs as ["--broken-diff"]. - # Avoid an error 500 by ignoring bad elements. See: - # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776 - def valid_raw_diff?(raw) - return false unless raw.respond_to?(:each) - - raw.any? { |element| VALID_CLASSES.include?(element.class) } - end - def create_merge_request_diff_files(diffs) rows = diffs.map.with_index do |diff, index| diff_hash = diff.to_hash.merge( @@ -256,9 +223,7 @@ class MergeRequestDiff < ActiveRecord::Base end def load_diffs(options) - return Gitlab::Git::DiffCollection.new([]) unless diffs_from_database - - raw = diffs_from_database + raw = merge_request_diff_files.map(&:to_hash) if paths = options[:paths] raw = raw.select do |diff| @@ -269,23 +234,11 @@ class MergeRequestDiff < ActiveRecord::Base Gitlab::Git::DiffCollection.new(raw, options) end - def diffs_from_database - return @diffs_from_database if defined?(@diffs_from_database) - - @diffs_from_database = - if st_diffs.present? - if valid_raw_diff?(st_diffs) - st_diffs - end - elsif merge_request_diff_files.present? - merge_request_diff_files.map(&:to_hash) - end - end - def load_commits - commits = st_commits.presence || merge_request_diff_commits + commits = merge_request_diff_commits.map { |commit| Commit.from_hash(commit.to_hash, project) } - commits.map { |commit| Commit.from_hash(commit.to_hash, project) } + CommitCollection + .new(merge_request.source_project, commits, merge_request.source_branch) end def save_diffs diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 47e6b785c39..c06ee8083f0 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -13,6 +13,7 @@ class Milestone < ActiveRecord::Base include Referable include StripAttribute include Milestoneish + include Gitlab::SQL::Pattern cache_markdown_field :title, pipeline: :single_line cache_markdown_field :description @@ -73,10 +74,7 @@ class Milestone < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:title].matches(pattern).or(t[:description].matches(pattern))) + fuzzy_search(query, [:title, :description]) end def filter_by_state(milestones, state) @@ -162,18 +160,18 @@ class Milestone < ActiveRecord::Base # Milestone.first.to_reference(cross_namespace_project) # => "gitlab-org/gitlab-ce%1" # Milestone.first.to_reference(same_namespace_project) # => "gitlab-ce%1" # - def to_reference(from_project = nil, format: :name, full: false) + def to_reference(from = nil, format: :name, full: false) format_reference = milestone_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" if project - "#{project.to_reference(from_project, full: full)}#{reference}" + "#{project.to_reference(from, full: full)}#{reference}" else reference end end - def reference_link_text(from_project = nil) + def reference_link_text(from = nil) self.title end @@ -256,7 +254,7 @@ class Milestone < ActiveRecord::Base def start_date_should_be_less_than_due_date if due_date <= start_date - errors.add(:start_date, "Can't be greater than due date") + errors.add(:due_date, "must be greater than start date") end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 4d401e7ba18..901dbf2ba69 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base include Routable include AfterCommitQueue include Storage::LegacyNamespace + include Gitlab::SQL::Pattern # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of @@ -86,10 +87,7 @@ class Namespace < ActiveRecord::Base # # Returns an ActiveRecord::Relation def search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:name].matches(pattern).or(t[:path].matches(pattern))) + fuzzy_search(query, [:name, :path]) end def clean_path(path) @@ -141,7 +139,17 @@ class Namespace < ActiveRecord::Base def find_fork_of(project) return nil unless project.fork_network - project.fork_network.find_forks_in(projects).first + if RequestStore.active? + forks_in_namespace = RequestStore.fetch("namespaces:#{id}:forked_projects") do + Hash.new do |found_forks, project| + found_forks[project] = project.fork_network.find_forks_in(projects).first + end + end + + forks_in_namespace[project] + else + project.fork_network.find_forks_in(projects).first + end end def lfs_enabled? diff --git a/app/models/note.rb b/app/models/note.rb index f9676361072..340fe087f82 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -14,6 +14,7 @@ class Note < ActiveRecord::Base include ResolvableNote include IgnorableColumn include Editable + include Gitlab::SQL::Pattern module SpecialRole FIRST_TIME_CONTRIBUTOR = :first_time_contributor @@ -110,6 +111,7 @@ class Note < ActiveRecord::Base includes(:author, :noteable, :updated_by, project: [:project_members, { group: [:group_members] }]) end + scope :with_metadata, -> { includes(:system_note_metadata) } after_initialize :ensure_discussion_id before_validation :nullify_blank_type, :nullify_blank_line_code @@ -166,10 +168,20 @@ class Note < ActiveRecord::Base def has_special_role?(role, note) note.special_role == role end + + def search(query) + fuzzy_search(query, [:note]) + end end def cross_reference? - system? && matches_cross_reference_regex? + return unless system? + + if force_cross_reference_regex_check? + matches_cross_reference_regex? + else + SystemNoteService.cross_reference?(note) + end end def diff_note? @@ -382,4 +394,10 @@ class Note < ActiveRecord::Base def set_discussion_id self.discussion_id ||= discussion_class.discussion_id(self) end + + def force_cross_reference_regex_check? + return unless system? + + SystemNoteMetadata::TYPES_WITH_CROSS_REFERENCES.include?(system_note_metadata&.action) + end end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 43c77f3f2a2..8de42ff9d2e 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -65,6 +65,7 @@ class PagesDomain < ActiveRecord::Base def expired? return false unless x509 + current = Time.new current < x509.not_before || x509.not_after < current end @@ -75,6 +76,7 @@ class PagesDomain < ActiveRecord::Base def subject return unless x509 + x509.subject.to_s end @@ -102,6 +104,7 @@ class PagesDomain < ActiveRecord::Base def validate_pages_domain return unless domain + if domain.downcase.ends_with?(Settings.pages.host.downcase) self.errors.add(:domain, "*.#{Settings.pages.host} is restricted") end @@ -109,6 +112,7 @@ class PagesDomain < ActiveRecord::Base def x509 return unless certificate + @x509 ||= OpenSSL::X509::Certificate.new(certificate) rescue OpenSSL::X509::CertificateError nil @@ -116,6 +120,7 @@ class PagesDomain < ActiveRecord::Base def pkey return unless key + @pkey ||= OpenSSL::PKey::RSA.new(key) rescue OpenSSL::PKey::PKeyError, OpenSSL::Cipher::CipherError nil diff --git a/app/models/project.rb b/app/models/project.rb index 894ded2a9f6..41657c171e2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -18,6 +18,7 @@ class Project < ActiveRecord::Base include SelectForProjectAuthorization include Routable include GroupDescendant + include Gitlab::SQL::Pattern extend Gitlab::ConfigHelper extend Gitlab::CurrentSettings @@ -188,7 +189,6 @@ class Project < ActiveRecord::Base has_one :statistics, class_name: 'ProjectStatistics' has_one :cluster_project, class_name: 'Clusters::Project' - has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster' has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' # Container repositories need to remove data from the container registry, @@ -233,8 +233,8 @@ class Project < ActiveRecord::Base validates :creator, presence: true, on: :create validates :description, length: { maximum: 2000 }, allow_blank: true validates :ci_config_path, - format: { without: /\.{2}/, - message: 'cannot include directory traversal.' }, + format: { without: /(\.{2}|\A\/)/, + message: 'cannot include leading slash or directory traversal.' }, length: { maximum: 255 }, allow_blank: true validates :name, @@ -272,8 +272,9 @@ class Project < ActiveRecord::Base scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } - scope :with_hashed_storage, -> { where('storage_version >= 1') } - scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) } + scope :with_storage_feature, ->(feature) { where('storage_version >= :version', version: HASHED_STORAGE_FEATURES[feature]) } + scope :without_storage_feature, ->(feature) { where('storage_version < :version OR storage_version IS NULL', version: HASHED_STORAGE_FEATURES[feature]) } + scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) } scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) } scope :sorted_by_stars, -> { reorder('projects.star_count DESC') } @@ -424,32 +425,11 @@ class Project < ActiveRecord::Base # # query - The search query as a String. def search(query) - ptable = arel_table - ntable = Namespace.arel_table - pattern = "%#{query}%" - - # unscoping unnecessary conditions that'll be applied - # when executing `where("projects.id IN (#{union.to_sql})")` - projects = unscoped.select(:id).where( - ptable[:path].matches(pattern) - .or(ptable[:name].matches(pattern)) - .or(ptable[:description].matches(pattern)) - ) - - namespaces = unscoped.select(:id) - .joins(:namespace) - .where(ntable[:name].matches(pattern)) - - union = Gitlab::SQL::Union.new([projects, namespaces]) - - where("projects.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + fuzzy_search(query, [:path, :name, :description]) end def search_by_title(query) - pattern = "%#{query}%" - table = Project.arel_table - - non_archived.where(table[:name].matches(pattern)) + non_archived.fuzzy_search(query, [:name]) end def visibility_levels @@ -581,8 +561,7 @@ class Project < ActiveRecord::Base if forked? RepositoryForkWorker.perform_async(id, forked_from_project.repository_storage_path, - forked_from_project.full_path, - self.namespace.full_path) + forked_from_project.disk_path) else RepositoryImportWorker.perform_async(self.id) end @@ -618,7 +597,7 @@ class Project < ActiveRecord::Base def ci_config_path=(value) # Strip all leading slashes so that //foo -> foo - super(value&.sub(%r{\A/+}, '')&.delete("\0")) + super(value&.delete("\0")) end def import_url=(value) @@ -760,10 +739,10 @@ class Project < ActiveRecord::Base end end - def to_human_reference(from_project = nil) - if cross_namespace_reference?(from_project) + def to_human_reference(from = nil) + if cross_namespace_reference?(from) name_with_namespace - elsif cross_project_reference?(from_project) + elsif cross_project_reference?(from) name end end @@ -772,13 +751,14 @@ class Project < ActiveRecord::Base Gitlab::Routing.url_helpers.project_url(self) end - def new_issue_address(author) + def new_issuable_address(author, address_type) return unless Gitlab::IncomingEmail.supports_issue_creation? && author author.ensure_incoming_email_token! + suffix = address_type == 'merge_request' ? '+merge-request' : '' Gitlab::IncomingEmail.reply_address( - "#{full_path}+#{author.incoming_email_token}") + "#{full_path}#{suffix}+#{author.incoming_email_token}") end def build_commit_note(commit) @@ -916,12 +896,10 @@ class Project < ActiveRecord::Base @ci_service ||= ci_services.reorder(nil).find_by(active: true) end - def deployment_services - services.where(category: :deployment) - end - - def deployment_service - @deployment_service ||= deployment_services.reorder(nil).find_by(active: true) + # TODO: This will be extended for multiple enviroment clusters + def deployment_platform + @deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes + @deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true) end def monitoring_services @@ -1135,7 +1113,11 @@ class Project < ActiveRecord::Base end def project_member(user) - project_members.find_by(user_id: user) + if project_members.loaded? + project_members.find { |member| member.user_id == user.id } + else + project_members.find_by(user_id: user) + end end def default_branch @@ -1566,9 +1548,9 @@ class Project < ActiveRecord::Base end def deployment_variables - return [] unless deployment_service + return [] unless deployment_platform - deployment_service.predefined_variables + deployment_platform.predefined_variables end def auto_devops_variables diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index 976d85246a8..768f0a7472e 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -51,8 +51,10 @@ class HipchatService < Service def execute(data) return unless supported_events.include?(data[:object_kind]) + message = create_message(data) return unless message.present? + gate[room].send('GitLab', message, message_options(data)) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index b487378edd2..1c065e1ddbd 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -176,6 +176,7 @@ class JiraService < IssueTrackerService def test_settings return unless client_url.present? + # Test settings by getting the project jira_request { client.ServerInfo.all.attrs } end diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 5080acffb3c..b82567ce2b3 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -1,3 +1,8 @@ +## +# NOTE: +# We'll move this class to Clusters::Platforms::Kubernetes, which contains exactly the same logic. +# After we've migrated data, we'll remove KubernetesService. This would happen in a few months. +# If you're modyfiyng this class, please note that you should update the same change in Clusters::Platforms::Kubernetes. class KubernetesService < DeploymentService include Gitlab::CurrentSettings include Gitlab::Kubernetes @@ -182,6 +187,7 @@ class KubernetesService < DeploymentService kubeclient.get_pods(namespace: actual_namespace).as_json rescue KubeException => err raise err unless err.error_code == 404 + [] end diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 715b215d1db..17b9d2cf7b4 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -35,7 +35,9 @@ class ProjectStatistics < ActiveRecord::Base end def update_build_artifacts_size - self.build_artifacts_size = project.builds.sum(:artifacts_size) + self.build_artifacts_size = + project.builds.sum(:artifacts_size) + + Ci::JobArtifact.artifacts_size_for(self) end def update_storage_size diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 3eecbea8cbf..a0af749a93f 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -76,8 +76,8 @@ class ProjectWiki # Returns an Array of Gitlab WikiPage instances or an # empty Array if this Wiki has no pages. - def pages - wiki.pages.map { |page| WikiPage.new(self, page, true) } + def pages(limit: nil) + wiki.pages(limit: limit).map { |page| WikiPage.new(self, page, true) } end # Finds a page within the repository based on a tile diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 89bfc5f9a9c..d28fed11ca8 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -10,7 +10,9 @@ class ProtectedBranch < ActiveRecord::Base def self.protected?(project, ref_name) return true if project.empty_repo? && default_branch_protected? - self.matching(ref_name, protected_refs: project.protected_branches).present? + refs = project.protected_branches.select(:name) + + self.matching(ref_name, protected_refs: refs).present? end def self.default_branch_protected? diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb index f38109c0e52..42a9bcf7723 100644 --- a/app/models/protected_tag.rb +++ b/app/models/protected_tag.rb @@ -5,6 +5,8 @@ class ProtectedTag < ActiveRecord::Base protected_ref_access_levels :create def self.protected?(project, ref_name) - self.matching(ref_name, protected_refs: project.protected_tags).present? + refs = project.protected_tags.select(:name) + + self.matching(ref_name, protected_refs: refs).present? end end diff --git a/app/models/protected_tag/create_access_level.rb b/app/models/protected_tag/create_access_level.rb index c7e1319719d..6b6ab3d8279 100644 --- a/app/models/protected_tag/create_access_level.rb +++ b/app/models/protected_tag/create_access_level.rb @@ -1,18 +1,6 @@ class ProtectedTag::CreateAccessLevel < ActiveRecord::Base include ProtectedTagAccess - validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER, - Gitlab::Access::DEVELOPER, - Gitlab::Access::NO_ACCESS] } - - def self.human_access_levels - { - Gitlab::Access::MASTER => "Masters", - Gitlab::Access::DEVELOPER => "Developers + Masters", - Gitlab::Access::NO_ACCESS => "No one" - }.with_indifferent_access - end - def check_access(user) return false if access_level == Gitlab::Access::NO_ACCESS diff --git a/app/models/repository.rb b/app/models/repository.rb index 3a89fa9264b..82af299ec5e 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -132,7 +132,8 @@ class Repository commits = Gitlab::Git::Commit.where(options) commits = Commit.decorate(commits, @project) if commits.present? - commits + + CommitCollection.new(project, commits, ref) end def commits_between(from, to) @@ -148,11 +149,14 @@ class Repository end raw_repository.gitaly_migrate(:commits_by_message) do |is_enabled| - if is_enabled - find_commits_by_message_by_gitaly(query, ref, path, limit, offset) - else - find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) - end + commits = + if is_enabled + find_commits_by_message_by_gitaly(query, ref, path, limit, offset) + else + find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) + end + + CommitCollection.new(project, commits, ref) end end @@ -213,11 +217,7 @@ class Repository def branch_exists?(branch_name) return false unless raw_repository - @branch_exists_memo ||= Hash.new do |hash, key| - hash[key] = raw_repository.branch_exists?(key) - end - - @branch_exists_memo[branch_name] + branch_names.include?(branch_name) end def ref_exists?(ref) @@ -242,6 +242,7 @@ class Repository Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" rescue Rugged::OSError => ex raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ + Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" end end @@ -255,7 +256,7 @@ class Repository end def diverging_commit_counts(branch) - root_ref_hash = raw_repository.rev_parse_target(root_ref).oid + root_ref_hash = raw_repository.commit(root_ref).id cache.fetch(:"diverging_commit_counts_#{branch.name}") do # Rugged seems to throw a `ReferenceError` when given branch_names rather # than SHA-1 hashes @@ -473,6 +474,11 @@ class Repository nil end + # items is an Array like: [[oid, path], [oid1, path1]] + def blobs_at(items) + raw_repository.batch_blobs(items).map { |blob| Blob.decorate(blob, project) } + end + def root_ref if raw_repository raw_repository.root_ref @@ -662,6 +668,7 @@ class Repository def next_branch(name, opts = {}) branch_ids = self.branch_names.map do |n| next 1 if n == name + result = n.match(/\A#{name}-([0-9]+)\z/) result[1].to_i if result end.compact @@ -902,19 +909,13 @@ class Repository end end - def merged_to_root_ref?(branch_or_name, pre_loaded_merged_branches = nil) + def merged_to_root_ref?(branch_or_name) branch = Gitlab::Git::Branch.find(self, branch_or_name) if branch @root_ref_sha ||= commit(root_ref).sha same_head = branch.target == @root_ref_sha - merged = - if pre_loaded_merged_branches - pre_loaded_merged_branches.include?(branch.name) - else - ancestor?(branch.target, @root_ref_sha) - end - + merged = ancestor?(branch.target, @root_ref_sha) !same_head && merged else nil @@ -965,6 +966,19 @@ class Repository run_git(args).first.lines.map(&:strip) end + def fetch_as_mirror(url, forced: false, refmap: :all_refs, remote_name: nil) + unless remote_name + remote_name = "tmp-#{SecureRandom.hex}" + tmp_remote_name = true + end + + add_remote(remote_name, url) + set_remote_as_mirror(remote_name, refmap: refmap) + fetch_remote(remote_name, forced: forced) + ensure + remove_remote(remote_name) if tmp_remote_name + end + def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false) gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) end @@ -990,10 +1004,6 @@ class Repository raw_repository.ls_files(actual_ref) end - def gitattribute(path, name) - raw_repository.attributes(path)[name] - end - def copy_gitattributes(ref) actual_ref = ref || root_ref begin @@ -1066,6 +1076,10 @@ class Repository raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref) end + def repository_storage_path + @project.repository_storage_path + end + private # TODO Generice finder, later split this on finders by Ref or Oid @@ -1131,10 +1145,6 @@ class Repository raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip end - def repository_storage_path - @project.repository_storage_path - end - def initialize_raw_repository Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki)) end diff --git a/app/models/service.rb b/app/models/service.rb index fdd2605e3e3..3c4f1885dd0 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -211,7 +211,7 @@ class Service < ActiveRecord::Base def async_execute(data) return unless supported_events.include?(data[:object_kind]) - Sidekiq::Client.enqueue(ProjectServiceWorker, id, data) + ProjectServiceWorker.perform_async(id, data) end def issue_tracker? diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 9533aa7f555..05a16f11b59 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -9,6 +9,7 @@ class Snippet < ActiveRecord::Base include Mentionable include Spammable include Editable + include Gitlab::SQL::Pattern extend Gitlab::CurrentSettings @@ -75,11 +76,11 @@ class Snippet < ActiveRecord::Base @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) end - def to_reference(from_project = nil, full: false) + def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{id}" if project.present? - "#{project.to_reference(from_project, full: full)}#{reference}" + "#{project.to_reference(from, full: full)}#{reference}" else reference end @@ -135,10 +136,7 @@ class Snippet < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:title].matches(pattern).or(t[:file_name].matches(pattern))) + fuzzy_search(query, [:title, :file_name]) end # Searches for snippets with matching content. @@ -149,10 +147,7 @@ class Snippet < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search_code(query) - table = Snippet.arel_table - pattern = "%#{query}%" - - where(table[:content].matches(pattern)) + fuzzy_search(query, [:content]) end end end diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index f025f40994e..fae1b64961a 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -4,7 +4,6 @@ module Storage delegate :gitlab_shell, :repository_storage_path, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze - STORAGE_VERSION = 1 def initialize(project) @project = project diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 1f9f8d7286b..29035480371 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -1,4 +1,14 @@ class SystemNoteMetadata < ActiveRecord::Base + # These notes's action text might contain a reference that is external. + # We should always force a deep validation upon references that are found + # in this note type. + # Other notes can always be safely shown as all its references are + # in the same project (i.e. with the same permissions) + TYPES_WITH_CROSS_REFERENCES = %w[ + commit cross_reference + close duplicate + ].freeze + ICON_TYPES = %w[ commit description merge confidential visible label assignee cross_reference title time_tracking branch milestone discussion task moved diff --git a/app/models/user.rb b/app/models/user.rb index ea10e2854d6..38ee4ed50c1 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -7,6 +7,7 @@ class User < ActiveRecord::Base include Gitlab::ConfigHelper include Gitlab::CurrentSettings include Gitlab::SQL::Pattern + include AfterCommitQueue include Avatarable include Referable include Sortable @@ -170,6 +171,7 @@ class User < ActiveRecord::Base after_save :ensure_namespace_correct after_update :username_changed_hook, if: :username_changed? after_destroy :post_destroy_hook + after_destroy :remove_key_cache after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') } after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') } @@ -268,8 +270,7 @@ class User < ActiveRecord::Base end def for_github_id(id) - joins(:identities) - .where(identities: { provider: :github, extern_uid: id.to_s }) + joins(:identities).merge(Identity.with_extern_uid(:github, id)) end # Find a User by their primary email or any associated secondary email @@ -313,9 +314,6 @@ class User < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search(query) - table = arel_table - pattern = User.to_pattern(query) - order = <<~SQL CASE WHEN users.name = %{query} THEN 0 @@ -325,11 +323,8 @@ class User < ActiveRecord::Base END SQL - where( - table[:name].matches(pattern) - .or(table[:email].matches(pattern)) - .or(table[:username].matches(pattern)) - ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) + fuzzy_search(query, [:name, :email, :username]) + .reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) end # searches user by given pattern @@ -337,16 +332,16 @@ class User < ActiveRecord::Base # This method uses ILIKE on PostgreSQL and LIKE on MySQL. def search_with_secondary_emails(query) - table = arel_table email_table = Email.arel_table - pattern = "%#{query}%" - matched_by_emails_user_ids = email_table.project(email_table[:user_id]).where(email_table[:email].matches(pattern)) + matched_by_emails_user_ids = email_table + .project(email_table[:user_id]) + .where(Email.fuzzy_arel_match(:email, query)) where( - table[:name].matches(pattern) - .or(table[:email].matches(pattern)) - .or(table[:username].matches(pattern)) - .or(table[:id].in(matched_by_emails_user_ids)) + fuzzy_arel_match(:name, query) + .or(fuzzy_arel_match(:email, query)) + .or(fuzzy_arel_match(:username, query)) + .or(arel_table[:id].in(matched_by_emails_user_ids)) ) end @@ -437,7 +432,7 @@ class User < ActiveRecord::Base username end - def to_reference(_from_project = nil, target_project: nil, full: nil) + def to_reference(_from = nil, target_project: nil, full: nil) "#{self.class.reference_prefix}#{username}" end @@ -445,6 +440,10 @@ class User < ActiveRecord::Base skip_confirmation! if bool end + def skip_reconfirmation=(bool) + skip_reconfirmation! if bool + end + def generate_reset_token @reset_token, enc = Devise.token_generator.generate(self.class, :reset_password_token) @@ -489,7 +488,11 @@ class User < ActiveRecord::Base end def two_factor_u2f_enabled? - u2f_registrations.exists? + if u2f_registrations.loaded? + u2f_registrations.any? + else + u2f_registrations.exists? + end end def namespace_uniq @@ -624,21 +627,39 @@ class User < ActiveRecord::Base end def require_ssh_key? - keys.count == 0 && Gitlab::ProtocolAccess.allowed?('ssh') + count = Users::KeysCountService.new(self).count + + count.zero? && Gitlab::ProtocolAccess.allowed?('ssh') + end + + def require_password_creation_for_web? + allow_password_authentication_for_web? && password_automatically_set? end - def require_password_creation? - password_automatically_set? && allow_password_authentication? + def require_password_creation_for_git? + allow_password_authentication_for_git? && password_automatically_set? end def require_personal_access_token_creation_for_git_auth? - return false if current_application_settings.password_authentication_enabled? || ldap_user? + return false if allow_password_authentication_for_git? || ldap_user? PersonalAccessTokensFinder.new(user: self, impersonation: false, state: 'active').execute.none? end + def require_extra_setup_for_git_auth? + require_password_creation_for_git? || require_personal_access_token_creation_for_git_auth? + end + def allow_password_authentication? - !ldap_user? && current_application_settings.password_authentication_enabled? + allow_password_authentication_for_web? || allow_password_authentication_for_git? + end + + def allow_password_authentication_for_web? + current_application_settings.password_authentication_enabled_for_web? && !ldap_user? + end + + def allow_password_authentication_for_git? + current_application_settings.password_authentication_enabled_for_git? && !ldap_user? end def can_change_username? @@ -883,9 +904,14 @@ class User < ActiveRecord::Base def post_destroy_hook log_info("User \"#{name}\" (#{email}) was removed") + system_hook_service.execute_hooks_for(self, :destroy) end + def remove_key_cache + Users::KeysCountService.new(self).delete_cache + end + def delete_async(deleted_by:, params: {}) block if params[:hard_delete] DeleteUserWorker.perform_async(deleted_by.id, id, params) @@ -978,7 +1004,11 @@ class User < ActiveRecord::Base end def notification_settings_for(source) - notification_settings.find_or_initialize_by(source: source) + if notification_settings.loaded? + notification_settings.find { |notification| notification.source == source } + else + notification_settings.find_or_initialize_by(source: source) + end end # Lazy load global notification setting @@ -1119,6 +1149,7 @@ class User < ActiveRecord::Base # override, from Devise::Validatable def password_required? return false if internal? + super end @@ -1136,6 +1167,7 @@ class User < ActiveRecord::Base # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration def send_devise_notification(notification, *args) return true unless can?(:receive_notifications) + devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 5f710961f95..bdfef677ef3 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -127,19 +127,24 @@ class WikiPage @version ||= @page.version end - # Returns an array of Gitlab Commit instances. - def versions + def versions(options = {}) return [] unless persisted? - wiki.wiki.page_versions(@page.path) + wiki.wiki.page_versions(@page.path, options) end - def commit - versions.first + def count_versions + return [] unless persisted? + + wiki.wiki.count_page_versions(@page.path) + end + + def last_version + @last_version ||= versions(limit: 1).first end def last_commit_sha - commit&.sha + last_version&.sha end # Returns the Date that this latest version was @@ -151,7 +156,7 @@ class WikiPage # Returns boolean True or False if this instance # is an old version of the page. def historical? - @page.historical? && versions.first.sha != version.sha + @page.historical? && last_version.sha != version.sha end # Returns boolean True or False if this instance |