diff options
| author | Jeremy Watson <jwatson@gitlab.com> | 2019-08-21 10:43:30 -0400 |
|---|---|---|
| committer | Jeremy Watson <jwatson@gitlab.com> | 2019-08-21 10:43:30 -0400 |
| commit | aec4ce4ac538992ae09c8a9c77be62a22ea239f1 (patch) | |
| tree | 0ec040a1d020a0096f60f1363db7fd1751967e9c /app/models | |
| parent | ad799726ae697d12664b8c3903e8297e7bfb4088 (diff) | |
| parent | ef0f1509dd2a2a3ba5798362e2be21108b705a85 (diff) | |
| download | gitlab-ce-docs-group-managed-accounts.tar.gz | |
Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into docs-group-managed-accountsdocs-group-managed-accounts
Diffstat (limited to 'app/models')
93 files changed, 1250 insertions, 740 deletions
diff --git a/app/models/active_session.rb b/app/models/active_session.rb index 345767179eb..00192b1da59 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -93,12 +93,12 @@ class ActiveSession end def self.list_sessions(user) - sessions_from_ids(session_ids_for_user(user)) + sessions_from_ids(session_ids_for_user(user.id)) end - def self.session_ids_for_user(user) + def self.session_ids_for_user(user_id) Gitlab::Redis::SharedState.with do |redis| - redis.smembers(lookup_key_name(user.id)) + redis.smembers(lookup_key_name(user_id)) end end @@ -129,15 +129,17 @@ class ActiveSession end def self.cleaned_up_lookup_entries(redis, user) - session_ids = session_ids_for_user(user) + session_ids = session_ids_for_user(user.id) entries = raw_active_session_entries(session_ids, user.id) # remove expired keys. # only the single key entries are automatically expired by redis, the # lookup entries in the set need to be removed manually. session_ids_and_entries = session_ids.zip(entries) - session_ids_and_entries.reject { |_session_id, entry| entry }.each do |session_id, _entry| - redis.srem(lookup_key_name(user.id), session_id) + redis.pipelined do + session_ids_and_entries.reject { |_session_id, entry| entry }.each do |session_id, _entry| + redis.srem(lookup_key_name(user.id), session_id) + end end entries.compact diff --git a/app/models/analytics/cycle_analytics.rb b/app/models/analytics/cycle_analytics.rb new file mode 100644 index 00000000000..626fc91cc41 --- /dev/null +++ b/app/models/analytics/cycle_analytics.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + def self.table_name_prefix + 'analytics_cycle_analytics_' + end + end +end diff --git a/app/models/analytics/cycle_analytics/project_stage.rb b/app/models/analytics/cycle_analytics/project_stage.rb new file mode 100644 index 00000000000..88c8cb40ccb --- /dev/null +++ b/app/models/analytics/cycle_analytics/project_stage.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + class ProjectStage < ApplicationRecord + belongs_to :project + end + end +end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 8e558487c1c..2a99c6e5c59 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -10,6 +10,8 @@ class ApplicationSetting < ApplicationRecord add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption, default_enabled: true) ? :optional : :required } add_authentication_token_field :health_check_access_token + belongs_to :instance_administration_project, class_name: "Project" + # Include here so it can override methods from # `add_authentication_token_field` # We don't prepend for now because otherwise we'll need to @@ -41,6 +43,11 @@ class ApplicationSetting < ApplicationRecord validates :uuid, presence: true + validates :outbound_local_requests_whitelist, + length: { maximum: 1_000, message: N_('is too long (maximum is 1000 entries)') }, + allow_nil: false, + qualified_domain_array: true + validates :session_expire_delay, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -92,6 +99,11 @@ class ApplicationSetting < ApplicationRecord presence: true, if: :plantuml_enabled + validates :snowplow_collector_hostname, + presence: true, + hostname: true, + if: :snowplow_enabled + validates :max_attachment_size, presence: true, numericality: { only_integer: true, greater_than: 0 } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index df4caed175d..55ac1e129cf 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -2,6 +2,7 @@ module ApplicationSettingImplementation extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace | # or @@ -20,7 +21,8 @@ module ApplicationSettingImplementation { after_sign_up_text: nil, akismet_enabled: false, - allow_local_requests_from_hooks_and_services: false, + allow_local_requests_from_web_hooks_and_services: false, + allow_local_requests_from_system_hooks: true, dns_rebinding_protection_enabled: true, authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand container_registry_token_expire_delay: 5, @@ -95,8 +97,14 @@ module ApplicationSettingImplementation usage_stats_set_by_user_id: nil, diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, commit_email_hostname: default_commit_email_hostname, + snowplow_collector_hostname: nil, + snowplow_cookie_domain: nil, + snowplow_enabled: false, + snowplow_site_id: nil, protected_ci_variables: false, - local_markdown_version: 0 + local_markdown_version: 0, + outbound_local_requests_whitelist: [], + raw_blob_request_limit: 300 } end @@ -131,31 +139,65 @@ module ApplicationSettingImplementation end def domain_whitelist_raw - self.domain_whitelist&.join("\n") + array_to_string(self.domain_whitelist) end def domain_blacklist_raw - self.domain_blacklist&.join("\n") + array_to_string(self.domain_blacklist) end def domain_whitelist_raw=(values) - self.domain_whitelist = [] - self.domain_whitelist = values.split(DOMAIN_LIST_SEPARATOR) - self.domain_whitelist.reject! { |d| d.empty? } - self.domain_whitelist + self.domain_whitelist = domain_strings_to_array(values) end def domain_blacklist_raw=(values) - self.domain_blacklist = [] - self.domain_blacklist = values.split(DOMAIN_LIST_SEPARATOR) - self.domain_blacklist.reject! { |d| d.empty? } - self.domain_blacklist + self.domain_blacklist = domain_strings_to_array(values) end def domain_blacklist_file=(file) self.domain_blacklist_raw = file.read end + def outbound_local_requests_whitelist_raw + array_to_string(self.outbound_local_requests_whitelist) + end + + def outbound_local_requests_whitelist_raw=(values) + clear_memoization(:outbound_local_requests_whitelist_arrays) + + self.outbound_local_requests_whitelist = domain_strings_to_array(values) + end + + def add_to_outbound_local_requests_whitelist(values_array) + clear_memoization(:outbound_local_requests_whitelist_arrays) + + self.outbound_local_requests_whitelist ||= [] + self.outbound_local_requests_whitelist += values_array + + self.outbound_local_requests_whitelist.uniq! + end + + def outbound_local_requests_whitelist_arrays + strong_memoize(:outbound_local_requests_whitelist_arrays) do + next [[], []] unless self.outbound_local_requests_whitelist + + ip_whitelist = [] + domain_whitelist = [] + + self.outbound_local_requests_whitelist.each do |str| + ip_obj = Gitlab::Utils.string_to_ip_object(str) + + if ip_obj + ip_whitelist << ip_obj + else + domain_whitelist << str + end + end + + [ip_whitelist, domain_whitelist] + end + end + def repository_storages Array(read_attribute(:repository_storages)) end @@ -255,6 +297,19 @@ module ApplicationSettingImplementation private + def array_to_string(arr) + arr&.join("\n") + end + + def domain_strings_to_array(values) + return [] unless values + + values + .split(DOMAIN_LIST_SEPARATOR) + .reject(&:empty?) + .uniq + end + def ensure_uuid! return if uuid? diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index e26162f6151..0ab302a0f3e 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -16,8 +16,10 @@ class AwardEmoji < ApplicationRecord participant :user - scope :downvotes, -> { where(name: DOWNVOTE_NAME) } - scope :upvotes, -> { where(name: UPVOTE_NAME) } + scope :downvotes, -> { named(DOWNVOTE_NAME) } + scope :upvotes, -> { named(UPVOTE_NAME) } + scope :named, -> (names) { where(name: names) } + scope :awarded_by, -> (users) { where(user: users) } after_save :expire_etag_cache after_destroy :expire_etag_cache diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 635fcc86166..3c0efca31db 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -38,8 +38,10 @@ module Ci has_one :deployment, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id + has_many :needs, class_name: 'Ci::BuildNeed', foreign_key: :build_id, inverse_of: :build has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent + has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id Ci::JobArtifact.file_types.each do |key, value| has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id @@ -48,6 +50,8 @@ module Ci has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build accepts_nested_attributes_for :runner_session + accepts_nested_attributes_for :job_variables + accepts_nested_attributes_for :needs delegate :url, to: :runner_session, prefix: true, allow_nil: true delegate :terminal_specification, to: :runner_session, allow_nil: true @@ -331,10 +335,10 @@ module Ci end # rubocop: disable CodeReuse/ServiceClass - def play(current_user) + def play(current_user, job_variables_attributes = nil) Ci::PlayBuildService .new(project, current_user) - .execute(self) + .execute(self, job_variables_attributes) end # rubocop: enable CodeReuse/ServiceClass @@ -380,7 +384,7 @@ module Ci return unless has_environment? strong_memoize(:expanded_environment_name) do - ExpandVariables.expand(environment, simple_variables) + ExpandVariables.expand(environment, -> { simple_variables }) end end @@ -432,6 +436,7 @@ module Ci Gitlab::Ci::Variables::Collection.new .concat(persisted_variables) .concat(scoped_variables) + .concat(job_variables) .concat(persisted_environment_variables) .to_runner_variables end @@ -531,6 +536,14 @@ module Ci trace.exist? end + def has_live_trace? + trace.live_trace_exist? + end + + def has_archived_trace? + trace.archived_trace_exist? + end + def artifacts_file job_artifacts_archive&.file end @@ -702,11 +715,17 @@ module Ci depended_jobs = depends_on_builds - return depended_jobs unless options[:dependencies].present? + # find all jobs that are needed + if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && needs.exists? + depended_jobs = depended_jobs.where(name: needs.select(:name)) + end - depended_jobs.select do |job| - options[:dependencies].include?(job.name) + # find all jobs that are dependent on + if options[:dependencies].present? + depended_jobs = depended_jobs.where(name: options[:dependencies]) end + + depended_jobs end def empty_dependencies? diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb new file mode 100644 index 00000000000..6531dfd332f --- /dev/null +++ b/app/models/ci/build_need.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + class BuildNeed < ApplicationRecord + extend Gitlab::Ci::Model + + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :needs + + validates :build, presence: true + validates :name, presence: true, length: { maximum: 128 } + + scope :scoped_build, -> { where('ci_builds.id=ci_build_needs.build_id') } + end +end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index f80e98e5bca..e132cb045e2 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -176,6 +176,10 @@ module Ci end end + def self.archived_trace_exists_for?(job_id) + where(job_id: job_id).trace.take&.file&.file&.exists? + end + private def file_format_adapter_class diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb new file mode 100644 index 00000000000..862a0bc1299 --- /dev/null +++ b/app/models/ci/job_variable.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + class JobVariable < ApplicationRecord + extend Gitlab::Ci::Model + include NewHasVariable + + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + + alias_attribute :secret_value, :value + + validates :key, uniqueness: { scope: :job_id } + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2262282e647..0a943a33bbb 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -229,15 +229,15 @@ module Ci # # ref - The name (or names) of the branch(es)/tag(s) to limit the list of # pipelines to. + # sha - The commit SHA (or mutliple SHAs) to limit the list of pipelines to. # limit - This limits a backlog search, default to 100. - def self.newest_first(ref: nil, limit: 100) + def self.newest_first(ref: nil, sha: nil, limit: 100) relation = order(id: :desc) relation = relation.where(ref: ref) if ref + relation = relation.where(sha: sha) if sha if limit ids = relation.limit(limit).select(:id) - # MySQL does not support limit in subquery - ids = ids.pluck(:id) if Gitlab::Database.mysql? relation = relation.where(id: ids) end @@ -248,10 +248,14 @@ module Ci newest_first(ref: ref).pluck(:status).first end - def self.latest_successful_for(ref) + def self.latest_successful_for_ref(ref) newest_first(ref: ref).success.take end + def self.latest_successful_for_sha(sha) + newest_first(sha: sha).success.take + end + def self.latest_successful_for_refs(refs) relation = newest_first(ref: refs).success @@ -324,6 +328,10 @@ module Ci config_sources.values_at(:repository_source, :auto_devops_source, :unknown_source) end + def self.bridgeable_statuses + ::Ci::Pipeline::AVAILABLE_STATUSES - %w[created preparing pending] + end + def stages_count statuses.select(:stage).distinct.count end @@ -500,8 +508,9 @@ module Ci return [] unless config_processor strong_memoize(:stage_seeds) do - seeds = config_processor.stages_attributes.map do |attributes| - Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes) + seeds = config_processor.stages_attributes.inject([]) do |previous_stages, attributes| + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes, previous_stages) + previous_stages + [seed] end seeds.select(&:included?) @@ -607,8 +616,8 @@ module Ci end # rubocop: disable CodeReuse/ServiceClass - def process! - Ci::ProcessPipelineService.new(project, user).execute(self) + def process!(trigger_build_ids = nil) + Ci::ProcessPipelineService.new(project, user).execute(self, trigger_build_ids) end # rubocop: enable CodeReuse/ServiceClass diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 07d00503861..43ff874ac23 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -264,7 +264,7 @@ module Ci private def cleanup_runner_queue - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) end end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index a77bbef0fca..760872d3e6b 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -6,6 +6,7 @@ module Ci include HasVariable include Presentable include Maskable + prepend HasEnvironmentScope belongs_to :project diff --git a/app/models/clusters/applications/cert_manager.rb b/app/models/clusters/applications/cert_manager.rb index d6a7d1d2bdd..6bd7473c8ff 100644 --- a/app/models/clusters/applications/cert_manager.rb +++ b/app/models/clusters/applications/cert_manager.rb @@ -24,12 +24,6 @@ module Clusters 'stable/cert-manager' end - # We will implement this in future MRs. - # Need to reverse postinstall step - def allowed_to_uninstall? - false - end - def install_command Gitlab::Kubernetes::Helm::InstallCommand.new( name: 'certmanager', @@ -41,10 +35,44 @@ module Clusters ) end + def uninstall_command + Gitlab::Kubernetes::Helm::DeleteCommand.new( + name: 'certmanager', + rbac: cluster.platform_kubernetes_rbac?, + files: files, + postdelete: post_delete_script + ) + end + private def post_install_script - ["/usr/bin/kubectl create -f /data/helm/certmanager/config/cluster_issuer.yaml"] + ["kubectl create -f /data/helm/certmanager/config/cluster_issuer.yaml"] + end + + def post_delete_script + [ + delete_private_key, + delete_crd('certificates.certmanager.k8s.io'), + delete_crd('clusterissuers.certmanager.k8s.io'), + delete_crd('issuers.certmanager.k8s.io') + ].compact + end + + def private_key_name + @private_key_name ||= cluster_issuer_content.dig('spec', 'acme', 'privateKeySecretRef', 'name') + end + + def delete_private_key + return unless private_key_name.present? + + args = %W(secret -n #{Gitlab::Kubernetes::Helm::NAMESPACE} #{private_key_name} --ignore-not-found) + + Gitlab::Kubernetes::KubectlCmd.delete(*args) + end + + def delete_crd(definition) + Gitlab::Kubernetes::KubectlCmd.delete("crd", definition, "--ignore-not-found") end def cluster_issuer_file diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb index a83d06c4b00..455cf200fbc 100644 --- a/app/models/clusters/applications/helm.rb +++ b/app/models/clusters/applications/helm.rb @@ -14,6 +14,7 @@ module Clusters include ::Clusters::Concerns::ApplicationCore include ::Clusters::Concerns::ApplicationStatus + include ::Gitlab::Utils::StrongMemoize default_value_for :version, Gitlab::Kubernetes::Helm::HELM_VERSION @@ -29,11 +30,22 @@ module Clusters self.status = 'installable' if cluster&.platform_kubernetes_active? end - # We will implement this in future MRs. - # Basically we need to check all other applications are not installed - # first. + # It can only be uninstalled if there are no other applications installed + # or with intermitent installation statuses in the database. def allowed_to_uninstall? - false + strong_memoize(:allowed_to_uninstall) do + applications = nil + + Clusters::Cluster::APPLICATIONS.each do |application_name, klass| + next if application_name == 'helm' + + extra_apps = Clusters::Applications::Helm.where('EXISTS (?)', klass.select(1).where(cluster_id: cluster_id)) + + applications = applications ? applications.or(extra_apps) : extra_apps + end + + !applications.exists? + end end def install_command @@ -44,6 +56,14 @@ module Clusters ) end + def uninstall_command + Gitlab::Kubernetes::Helm::ResetCommand.new( + name: name, + files: files, + rbac: cluster.platform_kubernetes_rbac? + ) + end + def has_ssl? ca_key.present? && ca_cert.present? end diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb index 5df4812bd25..244fe738396 100644 --- a/app/models/clusters/applications/knative.rb +++ b/app/models/clusters/applications/knative.rb @@ -7,6 +7,7 @@ module Clusters REPOSITORY = 'https://storage.googleapis.com/triggermesh-charts'.freeze METRICS_CONFIG = 'https://storage.googleapis.com/triggermesh-charts/istio-metrics.yaml'.freeze FETCH_IP_ADDRESS_DELAY = 30.seconds + API_RESOURCES_PATH = 'config/knative/api_resources.yml' self.table_name = 'clusters_applications_knative' @@ -46,12 +47,6 @@ module Clusters { "domain" => hostname }.to_yaml end - # Handled in a new issue: - # https://gitlab.com/gitlab-org/gitlab-ce/issues/59369 - def allowed_to_uninstall? - false - end - def install_command Gitlab::Kubernetes::Helm::InstallCommand.new( name: name, @@ -76,10 +71,61 @@ module Clusters cluster.kubeclient.get_service('istio-ingressgateway', 'istio-system') end + def uninstall_command + Gitlab::Kubernetes::Helm::DeleteCommand.new( + name: name, + rbac: cluster.platform_kubernetes_rbac?, + files: files, + predelete: delete_knative_services_and_metrics, + postdelete: delete_knative_istio_leftovers + ) + end + private + def delete_knative_services_and_metrics + delete_knative_services + delete_knative_istio_metrics + end + + def delete_knative_services + cluster.kubernetes_namespaces.map do |kubernetes_namespace| + Gitlab::Kubernetes::KubectlCmd.delete("ksvc", "--all", "-n", kubernetes_namespace.namespace) + end + end + + def delete_knative_istio_leftovers + delete_knative_namespaces + delete_knative_and_istio_crds + end + + def delete_knative_namespaces + [ + Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "ns", "knative-serving"), + Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "ns", "knative-build") + ] + end + + def delete_knative_and_istio_crds + api_resources.map do |crd| + Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "crd", "#{crd}") + end + end + + # returns an array of CRDs to be postdelete since helm does not + # manage the CRDs it creates. + def api_resources + @api_resources ||= YAML.safe_load(File.read(Rails.root.join(API_RESOURCES_PATH))) + end + def install_knative_metrics - ["kubectl apply -f #{METRICS_CONFIG}"] if cluster.application_prometheus_available? + return [] unless cluster.application_prometheus_available? + + [Gitlab::Kubernetes::KubectlCmd.apply_file(METRICS_CONFIG)] + end + + def delete_knative_istio_metrics + return [] unless cluster.application_prometheus_available? + + [Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "-f", METRICS_CONFIG)] end def verify_cluster? diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 805c8a73f8c..f31a6b8b50e 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -59,6 +59,15 @@ module Clusters ) end + def uninstall_command + Gitlab::Kubernetes::Helm::DeleteCommand.new( + name: name, + rbac: cluster.platform_kubernetes_rbac?, + files: files, + predelete: delete_knative_istio_metrics + ) + end + # Returns a copy of files where the values of 'values.yaml' # are replaced by the argument. # @@ -74,7 +83,7 @@ module Clusters # ensures headers containing auth data are appended to original k8s client options options = kube_client.rest_client.options.merge(headers: kube_client.headers) - RestClient::Resource.new(proxy_url, options) + Gitlab::PrometheusClient.new(proxy_url, options) rescue Kubeclient::HttpError # If users have mistakenly set parameters or removed the depended clusters, # `proxy_url` could raise an exception because gitlab can not communicate with the cluster. @@ -95,7 +104,15 @@ module Clusters end def install_knative_metrics - ["kubectl apply -f #{Clusters::Applications::Knative::METRICS_CONFIG}"] if cluster.application_knative_available? + return [] unless cluster.application_knative_available? + + [Gitlab::Kubernetes::KubectlCmd.apply_file(Clusters::Applications::Knative::METRICS_CONFIG)] + end + + def delete_knative_istio_metrics + return [] unless cluster.application_knative_available? + + [Gitlab::Kubernetes::KubectlCmd.delete("-f", Clusters::Applications::Knative::METRICS_CONFIG)] end end end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 6ae8c3bd7f3..6533b7a186e 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.6.0'.freeze + VERSION = '0.7.0'.freeze self.table_name = 'clusters_applications_runners' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 8c044c86c47..97d39491b73 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -53,6 +53,7 @@ module Clusters validates :name, cluster_name: true validates :cluster_type, presence: true validates :domain, allow_blank: true, hostname: { allow_numeric_hostname: true } + validates :namespace_per_environment, inclusion: { in: [true, false] } validate :restrict_modification, on: :update validate :no_groups, unless: :group_type? @@ -100,22 +101,6 @@ module Clusters scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) } - scope :missing_kubernetes_namespace, -> (kubernetes_namespaces) do - subquery = kubernetes_namespaces.select('1').where('clusters_kubernetes_namespaces.cluster_id = clusters.id') - - where('NOT EXISTS (?)', subquery) - end - - scope :with_knative_installed, -> { joins(:application_knative).merge(Clusters::Applications::Knative.available) } - - scope :preload_knative, -> { - preload( - :kubernetes_namespaces, - :platform_kubernetes, - :application_knative - ) - } - def self.ancestor_clusters_for_clusterable(clusterable, hierarchy_order: :asc) return [] if clusterable.is_a?(Instance) @@ -161,16 +146,6 @@ module Clusters return platform_kubernetes if kubernetes? end - def all_projects - if project_type? - projects - elsif group_type? - first_group.all_projects - else - Project.none - end - end - def first_project strong_memoize(:first_project) do projects.first @@ -193,36 +168,15 @@ module Clusters platform_kubernetes.kubeclient if kubernetes? end - ## - # This is subtly different to #find_or_initialize_kubernetes_namespace_for_project - # below because it will ignore any namespaces that have not got a service account - # token. This provides a guarantee that any namespace selected here can be used - # for cluster operations - a namespace needs to have a service account configured - # before it it can be used. - # - # This is used for selecting a namespace to use when querying a cluster, or - # generating variables to pass to CI. - def kubernetes_namespace_for(project) - find_or_initialize_kubernetes_namespace_for_project( - project, scope: kubernetes_namespaces.has_service_account_token - ).namespace - end + def kubernetes_namespace_for(environment) + project = environment.project + persisted_namespace = Clusters::KubernetesNamespaceFinder.new( + self, + project: project, + environment_slug: environment.slug + ).execute - ## - # This is subtly different to #kubernetes_namespace_for because it will include - # namespaces that have yet to receive a service account token. This allows - # the namespace configuration process to be repeatable - if a namespace has - # already been created without a token we don't need to create another - # record entirely, just set the token on the pre-existing namespace. - # - # This is used for configuring cluster namespaces. - def find_or_initialize_kubernetes_namespace_for_project(project, scope: kubernetes_namespaces) - attributes = { project: project } - attributes[:cluster_project] = cluster_project if project_type? - - scope.find_or_initialize_by(attributes).tap do |namespace| - namespace.set_defaults - end + persisted_namespace&.namespace || Gitlab::Kubernetes::DefaultNamespace.new(self, project: project).from_environment_slug(environment.slug) end def allow_user_defined_namespace? @@ -241,10 +195,6 @@ module Clusters end end - def knative_services_finder(project) - @knative_services_finder ||= KnativeServicesFinder.new(self, project) - end - private def instance_domain diff --git a/app/models/clusters/clusters_hierarchy.rb b/app/models/clusters/clusters_hierarchy.rb index dab034b7234..5556fc8d3f0 100644 --- a/app/models/clusters/clusters_hierarchy.rb +++ b/app/models/clusters/clusters_hierarchy.rb @@ -46,7 +46,7 @@ module Clusters def group_clusters_base_query group_parent_id_alias = alias_as_column(groups[:parent_id], 'group_parent_id') - join_sources = ::Group.left_joins(:clusters).join_sources + join_sources = ::Group.left_joins(:clusters).arel.join_sources model .unscoped @@ -59,7 +59,7 @@ module Clusters def project_clusters_base_query projects = ::Project.arel_table project_parent_id_alias = alias_as_column(projects[:namespace_id], 'group_parent_id') - join_sources = ::Project.left_joins(:clusters).join_sources + join_sources = ::Project.left_joins(:clusters).arel.join_sources model .unscoped diff --git a/app/models/clusters/kubernetes_namespace.rb b/app/models/clusters/kubernetes_namespace.rb index b0c4900546e..69a2b99fcb6 100644 --- a/app/models/clusters/kubernetes_namespace.rb +++ b/app/models/clusters/kubernetes_namespace.rb @@ -9,12 +9,12 @@ module Clusters belongs_to :cluster_project, class_name: 'Clusters::Project' belongs_to :cluster, class_name: 'Clusters::Cluster' belongs_to :project, class_name: '::Project' + belongs_to :environment, optional: true has_one :platform_kubernetes, through: :cluster - before_validation :set_defaults - validates :namespace, presence: true validates :namespace, uniqueness: { scope: :cluster_id } + validates :environment_id, uniqueness: { scope: [:cluster_id, :project_id] }, allow_nil: true validates :service_account_name, presence: true @@ -27,6 +27,7 @@ module Clusters algorithm: 'aes-256-cbc' scope :has_service_account_token, -> { where.not(encrypted_service_account_token: nil) } + scope :with_environment_slug, -> (slug) { joins(:environment).where(environments: { slug: slug }) } def token_name "#{namespace}-token" @@ -42,34 +43,8 @@ module Clusters end end - def set_defaults - self.namespace ||= default_platform_kubernetes_namespace - self.namespace ||= default_project_namespace - self.service_account_name ||= default_service_account_name - end - private - def default_service_account_name - return unless namespace - - "#{namespace}-service-account" - end - - def default_platform_kubernetes_namespace - platform_kubernetes&.namespace.presence - end - - def default_project_namespace - Gitlab::NamespaceSanitizer.sanitize(project_slug) if project_slug - end - - def project_slug - return unless project - - "#{project.path}-#{project.id}".downcase - end - def kubeconfig to_kubeconfig( url: api_url, diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 9296c28776b..37614fbe3ca 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -51,11 +51,6 @@ module Clusters delegate :provided_by_user?, to: :cluster, allow_nil: true delegate :allow_user_defined_namespace?, to: :cluster, allow_nil: true - # This is just to maintain compatibility with KubernetesService, which - # will be removed in https://gitlab.com/gitlab-org/gitlab-ce/issues/39217. - # It can be removed once KubernetesService is gone. - delegate :kubernetes_namespace_for, to: :cluster, allow_nil: true - alias_method :active?, :enabled? enum_with_nil authorization_type: { @@ -66,7 +61,7 @@ module Clusters default_value_for :authorization_type, :rbac - def predefined_variables(project:) + def predefined_variables(project:, environment_name:) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'KUBE_URL', value: api_url) @@ -77,15 +72,14 @@ module Clusters end if !cluster.managed? - project_namespace = namespace.presence || "#{project.path}-#{project.id}".downcase + namespace = Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: project).from_environment_name(environment_name) variables - .append(key: 'KUBE_URL', value: api_url) .append(key: 'KUBE_TOKEN', value: token, public: false, masked: true) - .append(key: 'KUBE_NAMESPACE', value: project_namespace) - .append(key: 'KUBECONFIG', value: kubeconfig(project_namespace), public: false, file: true) + .append(key: 'KUBE_NAMESPACE', value: namespace) + .append(key: 'KUBECONFIG', value: kubeconfig(namespace), public: false, file: true) - elsif kubernetes_namespace = cluster.kubernetes_namespaces.has_service_account_token.find_by(project: project) + elsif kubernetes_namespace = find_persisted_namespace(project, environment_name: environment_name) variables.concat(kubernetes_namespace.predefined_variables) end @@ -111,6 +105,22 @@ module Clusters private + ## + # Environment slug can be predicted given an environment + # name, so even if the environment isn't persisted yet we + # still know what to look for. + def environment_slug(name) + Gitlab::Slug::Environment.new(name).generate + end + + def find_persisted_namespace(project, environment_name:) + Clusters::KubernetesNamespaceFinder.new( + cluster, + project: project, + environment_slug: environment_slug(environment_name) + ).execute + end + def kubeconfig(namespace) to_kubeconfig( url: api_url, diff --git a/app/models/commit.rb b/app/models/commit.rb index be37fa2e76f..0889ce7e287 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -346,7 +346,7 @@ class Commit if commits_in_merge_request.present? message_body << "" - commits_in_merge_request.reverse.each do |commit_in_merge| + commits_in_merge_request.reverse_each do |commit_in_merge| message_body << "#{commit_in_merge.short_id} #{commit_in_merge.title}" end end diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index be6f3e9c5b0..4be4d95b4a1 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -40,8 +40,23 @@ class CommitStatus < ApplicationRecord scope :ordered, -> { order(:name) } scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) } scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } + scope :before_stage, -> (index) { where('stage_idx < ?', index) } + scope :for_stage, -> (index) { where(stage_idx: index) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) } + scope :for_ids, -> (ids) { where(id: ids) } + + scope :with_needs, -> (names = nil) do + needs = Ci::BuildNeed.scoped_build.select(1) + needs = needs.where(name: names) if names + where('EXISTS (?)', needs).preload(:needs) + end + + scope :without_needs, -> (names = nil) do + needs = Ci::BuildNeed.scoped_build.select(1) + needs = needs.where(name: names) if names + where('NOT EXISTS (?)', needs) + end # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. @@ -116,7 +131,7 @@ class CommitStatus < ApplicationRecord commit_status.run_after_commit do if pipeline_id if complete? || manual? - PipelineProcessWorker.perform_async(pipeline_id) + PipelineProcessWorker.perform_async(pipeline_id, [id]) else PipelineUpdateWorker.perform_async(pipeline_id) end @@ -139,6 +154,18 @@ class CommitStatus < ApplicationRecord end end + def self.names + select(:name) + end + + def self.status_for_prior_stages(index) + before_stage(index).latest.status || 'success' + end + + def self.status_for_names(names) + where(name: names).latest.status || 'success' + end + def locking_enabled? will_save_change_to_status? end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index 14bc56f0eee..f229b42ade6 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -106,30 +106,6 @@ module Awardable end def awarded_emoji?(emoji_name, current_user) - award_emoji.where(name: emoji_name, user: current_user).exists? - end - - def create_award_emoji(name, current_user) - return unless emoji_awardable? - - award_emoji.create(name: normalize_name(name), user: current_user) - end - - def remove_award_emoji(name, current_user) - award_emoji.where(name: name, user: current_user).destroy_all # rubocop: disable DestroyAll - end - - def toggle_award_emoji(emoji_name, current_user) - if awarded_emoji?(emoji_name, current_user) - remove_award_emoji(emoji_name, current_user) - else - create_award_emoji(emoji_name, current_user) - end - end - - private - - def normalize_name(name) - Gitlab::Emoji.normalize_emoji_name(name) + award_emoji.named(emoji_name).awarded_by(current_user).exists? end end diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb index 53dff2adfc3..0c800621a55 100644 --- a/app/models/concerns/cacheable_attributes.rb +++ b/app/models/concerns/cacheable_attributes.rb @@ -5,6 +5,8 @@ module CacheableAttributes included do after_commit { self.class.expire } + + private_class_method :request_store_cache_key end class_methods do @@ -32,7 +34,11 @@ module CacheableAttributes end def cached - Gitlab::SafeRequestStore[:"#{name}_cached_attributes"] ||= retrieve_from_cache + Gitlab::SafeRequestStore[request_store_cache_key] ||= retrieve_from_cache + end + + def request_store_cache_key + :"#{name}_cached_attributes" end def retrieve_from_cache @@ -58,6 +64,7 @@ module CacheableAttributes end def expire + Gitlab::SafeRequestStore.delete(request_store_cache_key) cache_backend.delete(cache_key) rescue # Gracefully handle when Redis is not available. For example, diff --git a/app/models/concerns/case_sensitivity.rb b/app/models/concerns/case_sensitivity.rb index c93b6589ee7..abddbf1c7e3 100644 --- a/app/models/concerns/case_sensitivity.rb +++ b/app/models/concerns/case_sensitivity.rb @@ -40,14 +40,10 @@ module CaseSensitivity end def lower_value(value) - return value if Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new('LOWER', [Arel::Nodes.build_quoted(value)]) end def lower_column(column) - return column if Gitlab::Database.mysql? - column.lower end end diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 9eed9492b37..304cc71e9dc 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -29,6 +29,7 @@ module Ci def degenerate! self.class.transaction do self.update!(options: nil, yaml_variables: nil) + self.needs.all.delete_all self.metadata&.destroy end end diff --git a/app/models/concerns/descendant.rb b/app/models/concerns/descendant.rb deleted file mode 100644 index 4c436522122..00000000000 --- a/app/models/concerns/descendant.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Descendant - extend ActiveSupport::Concern - - class_methods do - def supports_nested_objects? - Gitlab::Database.postgresql? - end - end -end diff --git a/app/models/concerns/diff_positionable_note.rb b/app/models/concerns/diff_positionable_note.rb index 2d09eff0111..195d9e107c5 100644 --- a/app/models/concerns/diff_positionable_note.rb +++ b/app/models/concerns/diff_positionable_note.rb @@ -10,6 +10,8 @@ module DiffPositionableNote serialize :original_position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize serialize :position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize serialize :change_position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize + + validate :diff_refs_match_commit, if: :for_commit? end %i(original_position position change_position).each do |meth| @@ -71,4 +73,10 @@ module DiffPositionableNote self.position = result[:position] end end + + def diff_refs_match_commit + return if self.original_position.diff_refs == commit&.diff_refs + + errors.add(:commit_id, 'does not match the diff refs') + end end diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb index cfffd845e43..ed14b73ac1b 100644 --- a/app/models/concerns/group_descendant.rb +++ b/app/models/concerns/group_descendant.rb @@ -42,7 +42,7 @@ module GroupDescendant parent = child.parent exception = ArgumentError.new <<~MSG - parent: [GroupDescendant: #{parent.inspect}] was not preloaded for [#{child.inspect}]") + Parent was not preloaded for child when rendering group hierarchy. This error is not user facing, but causes a +1 query. MSG extras = { @@ -50,7 +50,7 @@ module GroupDescendant child: child.inspect, preloaded: preloaded.map(&:full_path) } - issue_url = 'https://gitlab.com/gitlab-org/gitlab-ce/issues/40785' + issue_url = 'https://gitlab.com/gitlab-org/gitlab-ce/issues/49404' Gitlab::Sentry.track_exception(exception, issue_url: issue_url, extra: extras) end diff --git a/app/models/concerns/has_environment_scope.rb b/app/models/concerns/has_environment_scope.rb new file mode 100644 index 00000000000..9553abe4dd3 --- /dev/null +++ b/app/models/concerns/has_environment_scope.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module HasEnvironmentScope + extend ActiveSupport::Concern + + prepended do + validates( + :environment_scope, + presence: true, + format: { with: ::Gitlab::Regex.environment_scope_regex, + message: ::Gitlab::Regex.environment_scope_regex_message } + ) + + ## + # Select rows which have a scope that matches the given environment name. + # Rows are ordered by relevance, by default. The most relevant row is + # placed at the end of a list. + # + # options: + # - relevant_only: (boolean) + # You can get the most relevant row only. Other rows are not be + # selected even if its scope matches the environment name. + # This is equivalent to using `#last` from SQL standpoint. + # + scope :on_environment, -> (environment_name, relevant_only: false) do + order_direction = relevant_only ? 'DESC' : 'ASC' + + where = <<~SQL + environment_scope IN (:wildcard, :environment_name) OR + :environment_name LIKE + #{::Gitlab::SQL::Glob.to_like('environment_scope')} + SQL + + order = <<~SQL + CASE environment_scope + WHEN :wildcard THEN 0 + WHEN :environment_name THEN 2 + ELSE 1 + END #{order_direction} + SQL + + values = { + wildcard: '*', + environment_name: environment_name + } + + sanitized_order_sql = sanitize_sql_array([order, values]) + + # The query is trying to find variables with scopes matching the + # current environment name. Suppose the environment name is + # 'review/app', and we have variables with environment scopes like: + # * variable A: review + # * variable B: review/app + # * variable C: review/* + # * variable D: * + # And the query should find variable B, C, and D, because it would + # try to convert the scope into a LIKE pattern for each variable: + # * A: review + # * B: review/app + # * C: review/% + # * D: % + # Note that we'll match % and _ literally therefore we'll escape them. + # In this case, B, C, and D would match. We also want to prioritize + # the exact matched name, and put * last, and everything else in the + # middle. So the order should be: D < C < B + relation = where(where, values) + .order(Arel.sql(sanitized_order_sql)) # `order` cannot escape for us! + + relation = relation.limit(1) if relevant_only + + relation + end + end + + def environment_scope=(new_environment_scope) + super(new_environment_scope.to_s.strip) + end +end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 27a5c3d5286..71ebb586c13 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -106,10 +106,15 @@ module HasStatus scope :running_or_pending, -> { with_status(:running, :pending) } scope :finished, -> { with_status(:success, :failed, :canceled) } scope :failed_or_canceled, -> { with_status(:failed, :canceled) } + scope :incomplete, -> { without_statuses(completed_statuses) } scope :cancelable, -> do where(status: [:running, :preparing, :pending, :created, :scheduled]) end + + scope :without_statuses, -> (names) do + with_status(all_state_names - names.to_a) + end end def started? diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 952de92cae1..e60b6497cb7 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -427,4 +427,11 @@ module Issuable def wipless_title_changed(old_title) old_title != title end + + ## + # Overridden on EE module + # + def supports_milestone? + respond_to?(:milestone_id) + end end diff --git a/app/models/concerns/maskable.rb b/app/models/concerns/maskable.rb index e0f2c41b836..d70e47bc4ff 100644 --- a/app/models/concerns/maskable.rb +++ b/app/models/concerns/maskable.rb @@ -7,9 +7,10 @@ module Maskable # * No escape characters # * No variables # * No spaces - # * Minimal length of 8 characters from the Base64 alphabets (RFC4648) + # * Minimal length of 8 characters + # * Characters must be from the Base64 alphabet (RFC4648) with the addition of @ and : # * Absolutely no fun is allowed - REGEX = /\A[a-zA-Z0-9_+=\/-]{8,}\z/.freeze + REGEX = /\A[a-zA-Z0-9_+=\/@:-]{8,}\z/.freeze included do validates :masked, inclusion: { in: [true, false] } diff --git a/app/models/concerns/new_has_variable.rb b/app/models/concerns/new_has_variable.rb new file mode 100644 index 00000000000..429bf496872 --- /dev/null +++ b/app/models/concerns/new_has_variable.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module NewHasVariable + extend ActiveSupport::Concern + include HasVariable + + included do + attr_encrypted :value, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + insecure_mode: false + end +end diff --git a/app/models/concerns/project_api_compatibility.rb b/app/models/concerns/project_api_compatibility.rb index cb00efb06df..631b2a11e9a 100644 --- a/app/models/concerns/project_api_compatibility.rb +++ b/app/models/concerns/project_api_compatibility.rb @@ -9,12 +9,10 @@ module ProjectAPICompatibility end def auto_devops_enabled=(value) - self.build_auto_devops if self.auto_devops&.enabled.nil? - self.auto_devops.update! enabled: value + (auto_devops || build_auto_devops).enabled = value end def auto_devops_deploy_strategy=(value) - self.build_auto_devops if self.auto_devops&.enabled.nil? - self.auto_devops.update! deploy_strategy: value + (auto_devops || build_auto_devops).deploy_strategy = value end end diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index c2542dbe743..9ac4722c6b1 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -14,10 +14,6 @@ module PrometheusAdapter raise NotImplementedError end - def prometheus_client_wrapper - Gitlab::PrometheusClient.new(prometheus_client) - end - def can_query? prometheus_client.present? end @@ -35,7 +31,7 @@ module PrometheusAdapter def calculate_reactive_cache(query_class_name, *args) return unless prometheus_client - data = Object.const_get(query_class_name, false).new(prometheus_client_wrapper).query(*args) + data = Object.const_get(query_class_name, false).new(prometheus_client).query(*args) { success: true, data: data, diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index af387c99f3d..0648b4a78e1 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -47,7 +47,7 @@ module ProtectedRef def access_levels_for_ref(ref, action:, protected_refs: nil) self.matching(ref, protected_refs: protected_refs) - .map(&:"#{action}_access_levels").flatten + .flat_map(&:"#{action}_access_levels") end # Returns all protected refs that match the given ref name. diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index e4fe46d722a..6d3c7a7ed68 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -1,5 +1,26 @@ # frozen_string_literal: true +# This module makes it possible to handle items as a list, where the order of items can be easily altered +# Requirements: +# +# - Only works for ActiveRecord models +# - relative_position integer field must present on the model +# - This module uses GROUP BY: the model should have a parent relation, example: project -> issues, project is the parent relation (issues table has a parent_id column) +# +# Setup like this in the body of your class: +# +# include RelativePositioning +# +# # base query used for the position calculation +# def self.relative_positioning_query_base(issue) +# where(deleted: false) +# end +# +# # column that should be used in GROUP BY +# def self.relative_positioning_parent_column +# :project_id +# end +# module RelativePositioning extend ActiveSupport::Concern @@ -8,12 +29,8 @@ module RelativePositioning MAX_POSITION = Gitlab::Database::MAX_INT_VALUE IDEAL_DISTANCE = 500 - included do - after_save :save_positionable_neighbours - end - class_methods do - def move_to_end(objects) + def move_nulls_to_end(objects) objects = objects.reject(&:relative_position) return if objects.empty? @@ -22,7 +39,7 @@ module RelativePositioning self.transaction do objects.each do |object| - relative_position = position_between(max_relative_position, MAX_POSITION) + relative_position = position_between(max_relative_position || START_POSITION, MAX_POSITION) object.relative_position = relative_position max_relative_position = relative_position object.save(touch: false) @@ -93,11 +110,12 @@ module RelativePositioning return move_after(before) unless after return move_before(after) unless before - # If there is no place to insert an issue we need to create one by moving the before issue closer - # to its predecessor. This process will recursively move all the predecessors until we have a place + # If there is no place to insert an item we need to create one by moving the item + # before this and all preceding items until there is a gap + before, after = after, before if after.relative_position < before.relative_position if (after.relative_position - before.relative_position) < 2 - before.move_before - @positionable_neighbours = [before] # rubocop:disable Gitlab/ModuleWithInstanceVariables + after.move_sequence_before + before.reset end self.relative_position = self.class.position_between(before.relative_position, after.relative_position) @@ -107,12 +125,8 @@ module RelativePositioning pos_before = before.relative_position pos_after = before.next_relative_position - if before.shift_after? - issue_to_move = self.class.in_parents(parent_ids).find_by!(relative_position: pos_after) - issue_to_move.move_after - @positionable_neighbours = [issue_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables - - pos_after = issue_to_move.relative_position + if pos_after && (pos_after - pos_before) < 2 + before.move_sequence_after end self.relative_position = self.class.position_between(pos_before, pos_after) @@ -122,12 +136,8 @@ module RelativePositioning pos_after = after.relative_position pos_before = after.prev_relative_position - if after.shift_before? - issue_to_move = self.class.in_parents(parent_ids).find_by!(relative_position: pos_before) - issue_to_move.move_before - @positionable_neighbours = [issue_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables - - pos_before = issue_to_move.relative_position + if pos_before && (pos_after - pos_before) < 2 + after.move_sequence_before end self.relative_position = self.class.position_between(pos_before, pos_after) @@ -141,46 +151,95 @@ module RelativePositioning self.relative_position = self.class.position_between(min_relative_position || START_POSITION, MIN_POSITION) end - # Indicates if there is an issue that should be shifted to free the place - def shift_after? - next_pos = next_relative_position - next_pos && (next_pos - relative_position) == 1 + # Moves the sequence before the current item to the middle of the next gap + # For example, we have 5 11 12 13 14 15 and the current item is 15 + # This moves the sequence 11 12 13 14 to 8 9 10 11 + def move_sequence_before + next_gap = find_next_gap_before + delta = optimum_delta_for_gap(next_gap) + + move_sequence(next_gap[:start], relative_position, -delta) end - # Indicates if there is an issue that should be shifted to free the place - def shift_before? - prev_pos = prev_relative_position - prev_pos && (relative_position - prev_pos) == 1 + # Moves the sequence after the current item to the middle of the next gap + # For example, we have 11 12 13 14 15 21 and the current item is 11 + # This moves the sequence 12 13 14 15 to 15 16 17 18 + def move_sequence_after + next_gap = find_next_gap_after + delta = optimum_delta_for_gap(next_gap) + + move_sequence(relative_position, next_gap[:start], delta) end private - # rubocop:disable Gitlab/ModuleWithInstanceVariables - def save_positionable_neighbours - return unless @positionable_neighbours + # Supposing that we have a sequence of items: 1 5 11 12 13 and the current item is 13 + # This would return: `{ start: 11, end: 5 }` + def find_next_gap_before + items_with_next_pos = scoped_items + .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position DESC) AS next_pos') + .where('relative_position <= ?', relative_position) + .order(relative_position: :desc) + + find_next_gap(items_with_next_pos).tap do |gap| + gap[:end] ||= MIN_POSITION + end + end + + # Supposing that we have a sequence of items: 13 14 15 20 24 and the current item is 13 + # This would return: `{ start: 15, end: 20 }` + def find_next_gap_after + items_with_next_pos = scoped_items + .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position ASC) AS next_pos') + .where('relative_position >= ?', relative_position) + .order(:relative_position) - status = @positionable_neighbours.all? { |issue| issue.save(touch: false) } - @positionable_neighbours = nil + find_next_gap(items_with_next_pos).tap do |gap| + gap[:end] ||= MAX_POSITION + end + end + + def find_next_gap(items_with_next_pos) + gap = self.class.from(items_with_next_pos, :items_with_next_pos) + .where('ABS(pos - next_pos) > 1 OR next_pos IS NULL') + .limit(1) + .pluck(:pos, :next_pos) + .first + + { start: gap[0], end: gap[1] } + end - status + def optimum_delta_for_gap(gap) + delta = ((gap[:start] - gap[:end]) / 2.0).abs.ceil + + [delta, IDEAL_DISTANCE].min + end + + def move_sequence(start_pos, end_pos, delta) + scoped_items + .where.not(id: self.id) + .where('relative_position BETWEEN ? AND ?', start_pos, end_pos) + .update_all("relative_position = relative_position + #{delta}") end - # rubocop:enable Gitlab/ModuleWithInstanceVariables def calculate_relative_position(calculation) # When calculating across projects, this is much more efficient than # MAX(relative_position) without the GROUP BY, due to index usage: # https://gitlab.com/gitlab-org/gitlab-ce/issues/54276#note_119340977 - relation = self.class - .in_parents(parent_ids) + relation = scoped_items .order(Gitlab::Database.nulls_last_order('position', 'DESC')) + .group(self.class.relative_positioning_parent_column) .limit(1) - .group(self.class.parent_column) relation = yield relation if block_given? relation - .pluck(self.class.parent_column, Arel.sql("#{calculation}(relative_position) AS position")) + .pluck(self.class.relative_positioning_parent_column, Arel.sql("#{calculation}(relative_position) AS position")) .first&. last end + + def scoped_items + self.class.relative_positioning_query_base(self) + end end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 9becab632f3..116e8967651 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -33,29 +33,12 @@ module Routable # # Returns a single object, or nil. def find_by_full_path(path, follow_redirects: false) - # On MySQL we want to ensure the ORDER BY uses a case-sensitive match so - # any literal matches come first, for this we have to use "BINARY". - # Without this there's still no guarantee in what order MySQL will return - # rows. - # - # Why do we do this? - # - # Even though we have Rails validation on Route for unique paths - # (case-insensitive), there are old projects in our DB (and possibly - # clients' DBs) that have the same path with different cases. - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/18603. Also note that - # our unique index is case-sensitive in Postgres. - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = Arel.sql("(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") + order_sql = Arel.sql("(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") found = where_full_path_in([path]).reorder(order_sql).take return found if found if follow_redirects - if Gitlab::Database.postgresql? - joins(:redirect_routes).find_by("LOWER(redirect_routes.path) = LOWER(?)", path) - else - joins(:redirect_routes).find_by(redirect_routes: { path: path }) - end + joins(:redirect_routes).find_by("LOWER(redirect_routes.path) = LOWER(?)", path) end end @@ -67,27 +50,13 @@ module Routable # # Returns an ActiveRecord::Relation. def where_full_path_in(paths) - wheres = [] - cast_lower = Gitlab::Database.postgresql? + return none if paths.empty? - paths.each do |path| - path = connection.quote(path) - - where = - if cast_lower - "(LOWER(routes.path) = LOWER(#{path}))" - else - "(routes.path = #{path})" - end - - wheres << where + wheres = paths.map do |path| + "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" end - if wheres.empty? - none - else - joins(:route).where(wheres.join(' OR ')) - end + joins(:route).where(wheres.join(' OR ')) end end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index b42adad94ba..8b536a123fc 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -15,7 +15,8 @@ module Taskable INCOMPLETE_PATTERN = /(\[[\s]\])/.freeze ITEM_PATTERN = %r{ ^ - \s*(?:[-+*]|(?:\d+\.)) # list prefix required - task item has to be always in a list + (?:(?:>\s{0,4})*) # optional blockquote characters + \s*(?:[-+*]|(?:\d+\.)) # list prefix required - task item has to be always in a list \s+ # whitespace prefix has to be always presented for a list item (\[\s\]|\[[xX]\]) # checkbox (\s.+) # followed by whitespace and some text. diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 1293df571a3..4099039dd96 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -3,10 +3,8 @@ module TokenAuthenticatable extend ActiveSupport::Concern - private - class_methods do - private # rubocop:disable Lint/UselessAccessModifier + private def add_authentication_token_field(token_field, options = {}) if token_authenticatable_fields.include?(token_field) diff --git a/app/models/concerns/update_project_statistics.rb b/app/models/concerns/update_project_statistics.rb index 570a735973f..869b3490f3f 100644 --- a/app/models/concerns/update_project_statistics.rb +++ b/app/models/concerns/update_project_statistics.rb @@ -73,15 +73,10 @@ module UpdateProjectStatistics def schedule_namespace_aggregation_worker run_after_commit do - next unless schedule_aggregation_worker? + next if project.nil? Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id) end end - - def schedule_aggregation_worker? - !project.nil? && - Feature.enabled?(:update_statistics_namespace, project.root_ancestor) - end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 39e12ac2b06..2a5ae7930e6 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -70,10 +70,14 @@ class ContainerRepository < ApplicationRecord digests = tags.map { |tag| tag.digest }.to_set digests.all? do |digest| - client.delete_repository_tag(self.path, digest) + delete_tag_by_digest(digest) end end + def delete_tag_by_digest(digest) + client.delete_repository_tag(self.path, digest) + end + def self.build_from_path(path) self.new(project: path.repository_project, name: path.repository_name) @@ -86,4 +90,9 @@ class ContainerRepository < ApplicationRecord def self.build_root_repository(project) self.new(project: project, name: '') end + + def self.find_by_path!(path) + self.find_by!(project: path.repository_project, + name: path.repository_name) + end end diff --git a/app/models/cycle_analytics/group_level.rb b/app/models/cycle_analytics/group_level.rb new file mode 100644 index 00000000000..a41e1375484 --- /dev/null +++ b/app/models/cycle_analytics/group_level.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module CycleAnalytics + class GroupLevel + include LevelBase + attr_reader :options, :group + + def initialize(group:, options:) + @group = group + @options = options.merge(group: group) + end + + def summary + @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group, options: options).data + end + + def permissions(*) + STAGES.each_with_object({}) do |stage, obj| + obj[stage] = true + end + end + + def stats + @stats ||= STAGES.map do |stage_name| + self[stage_name].as_json(serializer: GroupAnalyticsStageSerializer) + end + end + end +end diff --git a/app/models/cycle_analytics/base.rb b/app/models/cycle_analytics/level_base.rb index d7b28cd1b67..543349ebf8f 100644 --- a/app/models/cycle_analytics/base.rb +++ b/app/models/cycle_analytics/level_base.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true module CycleAnalytics - class Base + module LevelBase STAGES = %i[issue plan code test review staging production].freeze def all_medians_by_stage STAGES.each_with_object({}) do |stage_name, medians_per_stage| - medians_per_stage[stage_name] = self[stage_name].median + medians_per_stage[stage_name] = self[stage_name].project_median end end @@ -21,7 +21,7 @@ module CycleAnalytics end def [](stage_name) - Gitlab::CycleAnalytics::Stage[stage_name].new(project: @project, options: @options) + Gitlab::CycleAnalytics::Stage[stage_name].new(options: options) end end end diff --git a/app/models/cycle_analytics/project_level.rb b/app/models/cycle_analytics/project_level.rb index 22631cc7d41..4aa426c58a1 100644 --- a/app/models/cycle_analytics/project_level.rb +++ b/app/models/cycle_analytics/project_level.rb @@ -1,12 +1,13 @@ # frozen_string_literal: true module CycleAnalytics - class ProjectLevel < Base + class ProjectLevel + include LevelBase attr_reader :project, :options def initialize(project, options:) @project = project - @options = options + @options = options.merge(project: project) end def summary diff --git a/app/models/dashboard_group_milestone.rb b/app/models/dashboard_group_milestone.rb index 74aa04ab7d0..ec52f1ed370 100644 --- a/app/models/dashboard_group_milestone.rb +++ b/app/models/dashboard_group_milestone.rb @@ -15,8 +15,7 @@ class DashboardGroupMilestone < GlobalMilestone milestones = Milestone.of_groups(groups.select(:id)) .reorder_by_due_date_asc .order_by_name_asc - .active milestones = milestones.search_title(params[:search_title]) if params[:search_title].present? - milestones.map { |m| new(m) } + Milestone.filter_by_state(milestones, params[:state]).map { |m| new(m) } end end diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index db501b4b506..0bd90bd28e3 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -2,12 +2,14 @@ class DeployKey < Key include IgnorableColumn + include FromUnion has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :deploy_keys_projects scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where('deploy_keys_projects.project_id in (?)', projects) } scope :are_public, -> { where(public: true) } + scope :with_projects, -> { includes(deploy_keys_projects: { project: [:route, :namespace] }) } ignore_column :can_push @@ -22,7 +24,7 @@ class DeployKey < Key end def almost_orphaned? - self.deploy_keys_projects.length == 1 + self.deploy_keys_projects.count == 1 end def destroyed_when_orphaned? @@ -46,6 +48,6 @@ class DeployKey < Key end def projects_with_write_access - Project.preload(:route).where(id: deploy_keys_projects.with_write_access.select(:project_id)) + Project.with_route.where(id: deploy_keys_projects.with_write_access.select(:project_id)) end end diff --git a/app/models/deploy_keys_project.rb b/app/models/deploy_keys_project.rb index 15906ed8e06..40c66d5bc4c 100644 --- a/app/models/deploy_keys_project.rb +++ b/app/models/deploy_keys_project.rb @@ -1,9 +1,8 @@ # frozen_string_literal: true class DeployKeysProject < ApplicationRecord - belongs_to :project + belongs_to :project, inverse_of: :deploy_keys_projects belongs_to :deploy_key, inverse_of: :deploy_keys_projects - scope :without_project_deleted, -> { joins(:project).where(projects: { pending_delete: false }) } scope :in_project, ->(project) { where(project: project) } scope :with_write_access, -> { where(can_push: true) } diff --git a/app/models/deployment.rb b/app/models/deployment.rb index b69cda4f2f9..68586e7a1fd 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -128,17 +128,8 @@ class Deployment < ApplicationRecord merge_requests = merge_requests.where("merge_request_metrics.merged_at >= ?", previous_deployment.finished_at) end - # Need to use `map` instead of `select` because MySQL doesn't allow `SELECT`ing from the same table - # that we're updating. - merge_request_ids = - if Gitlab::Database.postgresql? - merge_requests.select(:id) - elsif Gitlab::Database.mysql? - merge_requests.map(&:id) - end - MergeRequest::Metrics - .where(merge_request_id: merge_request_ids, first_deployed_to_production_at: nil) + .where(merge_request_id: merge_requests.select(:id), first_deployed_to_production_at: nil) .update_all(first_deployed_to_production_at: finished_at) end diff --git a/app/models/deployment_metrics.rb b/app/models/deployment_metrics.rb index cfe762ca25e..2056c8bc59c 100644 --- a/app/models/deployment_metrics.rb +++ b/app/models/deployment_metrics.rb @@ -44,18 +44,7 @@ class DeploymentMetrics end end - # TODO remove fallback case to deployment_platform_cluster. - # Otherwise we will continue to pay the performance penalty described in - # https://gitlab.com/gitlab-org/gitlab-ce/issues/63475 - # - # Removal issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/64105 def cluster_prometheus - cluster_with_fallback = cluster || deployment_platform_cluster - - cluster_with_fallback.application_prometheus if cluster_with_fallback&.application_prometheus_available? - end - - def deployment_platform_cluster - deployment.environment.deployment_platform&.cluster + cluster.application_prometheus if cluster&.application_prometheus_available? end end diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index f75c32633b1..861185dc222 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -20,7 +20,6 @@ class DiffNote < Note validates :noteable_type, inclusion: { in: -> (_note) { noteable_types } } validate :positions_complete validate :verify_supported - validate :diff_refs_match_commit, if: :for_commit? before_validation :set_line_code, if: :on_text? after_save :keep_around_commits @@ -154,12 +153,6 @@ class DiffNote < Note errors.add(:position, "is invalid") end - def diff_refs_match_commit - return if self.original_position.diff_refs == self.commit.diff_refs - - errors.add(:commit_id, 'does not match the diff refs') - end - def keep_around_commits shas = [ self.original_position.base_sha, diff --git a/app/models/environment.rb b/app/models/environment.rb index b8ee54c1696..1b53c4b45f9 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -4,11 +4,6 @@ class Environment < ApplicationRecord include Gitlab::Utils::StrongMemoize include ReactiveCaching - # Used to generate random suffixes for the slug - LETTERS = ('a'..'z').freeze - NUMBERS = ('0'..'9').freeze - SUFFIX_CHARS = LETTERS.to_a + NUMBERS.to_a - belongs_to :project, required: true has_many :deployments, -> { success }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -53,6 +48,7 @@ class Environment < ApplicationRecord end scope :in_review_folder, -> { where(environment_type: "review") } scope :for_name, -> (name) { where(name: name) } + scope :preload_cluster, -> { preload(last_deployment: :cluster) } ## # Search environments which have names like the given query. @@ -175,7 +171,7 @@ class Environment < ApplicationRecord def deployment_namespace strong_memoize(:kubernetes_namespace) do - deployment_platform&.kubernetes_namespace_for(project) + deployment_platform.cluster.kubernetes_namespace_for(self) if deployment_platform end end @@ -203,47 +199,13 @@ class Environment < ApplicationRecord super.presence || generate_slug end - # An environment name is not necessarily suitable for use in URLs, DNS - # or other third-party contexts, so provide a slugified version. A slug has - # the following properties: - # * contains only lowercase letters (a-z), numbers (0-9), and '-' - # * begins with a letter - # * has a maximum length of 24 bytes (OpenShift limitation) - # * cannot end with `-` - def generate_slug - # Lowercase letters and numbers only - slugified = +name.to_s.downcase.gsub(/[^a-z0-9]/, '-') - - # Must start with a letter - slugified = 'env-' + slugified unless LETTERS.cover?(slugified[0]) - - # Repeated dashes are invalid (OpenShift limitation) - slugified.gsub!(/\-+/, '-') - - # Maximum length: 24 characters (OpenShift limitation) - slugified = slugified[0..23] - - # Cannot end with a dash (Kubernetes label limitation) - slugified.chop! if slugified.end_with?('-') - - # Add a random suffix, shortening the current string if necessary, if it - # has been slugified. This ensures uniqueness. - if slugified != name - slugified = slugified[0..16] - slugified << '-' unless slugified.end_with?('-') - slugified << random_suffix - end - - self.slug = slugified - end - def external_url_for(path, commit_sha) return unless self.external_url public_path = project.public_path_for_source_path(path, commit_sha) return unless public_path - [external_url, public_path].join('/') + [external_url.delete_suffix('/'), public_path.delete_prefix('/')].join('/') end def expire_etag_cache @@ -272,13 +234,15 @@ class Environment < ApplicationRecord end end + def knative_services_finder + if last_deployment&.cluster + Clusters::KnativeServicesFinder.new(last_deployment.cluster, self) + end + end + private - # Slugifying a name may remove the uniqueness guarantee afforded by it being - # based on name (which must be unique). To compensate, we add a random - # 6-byte suffix in those circumstances. This is not *guaranteed* uniqueness, - # but the chance of collisions is vanishingly small - def random_suffix - (0..5).map { SUFFIX_CHARS.sample }.join + def generate_slug + self.slug = Gitlab::Slug::Environment.new(name).generate end end diff --git a/app/models/group.rb b/app/models/group.rb index 9520db1bc0a..6c868b1d1f0 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -10,7 +10,6 @@ class Group < Namespace include Referable include SelectForProjectAuthorization include LoadedInGroupList - include Descendant include GroupDescendant include TokenAuthenticatable include WithUploads @@ -45,6 +44,8 @@ class Group < Namespace has_many :cluster_groups, class_name: 'Clusters::Group' has_many :clusters, through: :cluster_groups, class_name: 'Clusters::Cluster' + has_many :container_repositories, through: :projects + has_many :todos accepts_nested_attributes_for :variables, allow_destroy: true @@ -144,6 +145,12 @@ class Group < Namespace notification_settings(hierarchy_order: hierarchy_order).where(user: user) end + def notification_email_for(user) + # Finds the closest notification_setting with a `notification_email` + notification_settings = notification_settings_for(user, hierarchy_order: :asc) + notification_settings.find { |n| n.notification_email.present? }&.notification_email + end + def to_reference(_from = nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end @@ -382,7 +389,7 @@ class Group < Namespace variables = Ci::GroupVariable.where(group: list_of_ids) variables = variables.unprotected unless project.protected_for?(ref) variables = variables.group_by(&:group_id) - list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten + list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact end def group_member(user) @@ -416,6 +423,10 @@ class Group < Namespace super || ::Gitlab::CurrentSettings.default_project_creation end + def subgroup_creation_level + super || ::Gitlab::Access::OWNER_SUBGROUP_ACCESS + end + private def update_two_factor_requirement diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index 90b4588a325..3d54d17e787 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -14,8 +14,10 @@ class SystemHook < WebHook default_value_for :repository_update_events, true default_value_for :merge_requests_events, false + validates :url, system_hook_url: true + # Allow urls pointing localhost and the local network def allow_local_requests? - true + Gitlab::CurrentSettings.allow_local_requests_from_system_hooks? end end diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index daf7ff4b771..16fc7fdbd48 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -15,8 +15,8 @@ class WebHook < ApplicationRecord has_many :web_hook_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - validates :url, presence: true, public_url: { allow_localhost: lambda(&:allow_local_requests?), - allow_local_network: lambda(&:allow_local_requests?) } + validates :url, presence: true + validates :url, public_url: true, unless: ->(hook) { hook.is_a?(SystemHook) } validates :token, format: { without: /\n/ } validates :push_events_branch_filter, branch_filter: true @@ -35,6 +35,6 @@ class WebHook < ApplicationRecord # Allow urls pointing localhost and the local network def allow_local_requests? - false + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 12d30389910..c5a18f0af0f 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -43,6 +43,7 @@ class Issue < ApplicationRecord validates :project, presence: true alias_attribute :parent_ids, :project_id + alias_method :issuing_parent, :project scope :in_projects, ->(project_ids) { where(project_id: project_ids) } @@ -63,7 +64,7 @@ class Issue < ApplicationRecord scope :public_only, -> { where(confidential: false) } scope :confidential_only, -> { where(confidential: true) } - after_save :expire_etag_cache + after_commit :expire_etag_cache after_save :ensure_metrics, unless: :imported? attr_spammable :title, spam_title: true @@ -91,11 +92,11 @@ class Issue < ApplicationRecord end end - class << self - alias_method :in_parents, :in_projects + def self.relative_positioning_query_base(issue) + in_projects(issue.parent_ids) end - def self.parent_column + def self.relative_positioning_parent_column :project_id end @@ -131,7 +132,7 @@ class Issue < ApplicationRecord when 'due_date' then order_due_date_asc when 'due_date_asc' then order_due_date_asc when 'due_date_desc' then order_due_date_desc - when 'relative_position' then order_relative_position_asc + when 'relative_position' then order_relative_position_asc.with_order_id_desc else super end diff --git a/app/models/label.rb b/app/models/label.rb index b83e0862bab..d9455b36242 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -33,7 +33,7 @@ class Label < ApplicationRecord default_scope { order(title: :asc) } - scope :templates, -> { where(template: true) } + scope :templates, -> { where(template: true, type: [Label.name, nil]) } scope :with_title, ->(title) { where(title: title) } scope :with_lists_and_board, -> { joins(lists: :board).merge(List.movable) } scope :on_project_boards, ->(project_id) { with_lists_and_board.where(boards: { project_id: project_id }) } @@ -137,6 +137,12 @@ class Label < ApplicationRecord where(id: ids) end + def self.on_project_board?(project_id, label_id) + return false if label_id.blank? + + on_project_boards(project_id).where(id: label_id).exists? + end + def open_issues_count(user = nil) issues_count(user, state: 'opened') end diff --git a/app/models/list.rb b/app/models/list.rb index d28a9bda82d..ccadd39bda2 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -3,10 +3,11 @@ class List < ApplicationRecord belongs_to :board belongs_to :label + include Importable enum list_type: { backlog: 0, label: 1, closed: 2, assignee: 3, milestone: 4 } - validates :board, :list_type, presence: true + validates :board, :list_type, presence: true, unless: :importing? validates :label, :position, presence: true, if: :label? validates :label_id, uniqueness: { scope: :board_id }, if: :label? validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, if: :movable? diff --git a/app/models/member.rb b/app/models/member.rb index c7583434148..dbae1076670 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -107,6 +107,10 @@ class Member < ApplicationRecord joins(:user).merge(User.search(query)) end + def search_invite_email(query) + invite.where(['invite_email ILIKE ?', "%#{query}%"]) + end + def filter_by_2fa(value) case value when 'enabled' diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 4cba69069bb..3d6f397e599 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class GroupMember < Member + include FromUnion + SOURCE_TYPE = 'Namespace'.freeze belongs_to :group, foreign_key: 'source_id' @@ -13,8 +15,8 @@ class GroupMember < Member default_scope { where(source_type: SOURCE_TYPE) } scope :of_groups, ->(groups) { where(source_id: groups.select(:id)) } - scope :count_users_by_group_id, -> { joins(:user).group(:source_id).count } + scope :of_ldap_type, -> { where(ldap: true) } after_create :update_two_factor_requirement, unless: :invite? after_destroy :update_two_factor_requirement, unless: :invite? diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 68e6e48fb7d..bfd636fa62a 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -73,6 +73,7 @@ class MergeRequest < ApplicationRecord after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed after_save :ensure_metrics + after_commit :expire_etag_cache # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests @@ -192,6 +193,7 @@ class MergeRequest < ApplicationRecord alias_attribute :project, :target_project alias_attribute :project_id, :target_project_id alias_attribute :auto_merge_enabled, :merge_when_pipeline_succeeds + alias_method :issuing_parent, :target_project def self.reference_prefix '!' @@ -218,18 +220,7 @@ class MergeRequest < ApplicationRecord end def rebase_in_progress? - (rebase_jid.present? && Gitlab::SidekiqStatus.running?(rebase_jid)) || - gitaly_rebase_in_progress? - end - - # TODO: remove the Gitaly lookup after v12.1, when rebase_jid will be reliable - def gitaly_rebase_in_progress? - strong_memoize(:gitaly_rebase_in_progress) do - # The source project can be deleted - next false unless source_project - - source_project.repository.rebase_in_progress?(id) - end + rebase_jid.present? && Gitlab::SidekiqStatus.running?(rebase_jid) end # Use this method whenever you need to make sure the head_pipeline is synced with the @@ -388,6 +379,10 @@ class MergeRequest < ApplicationRecord def merge_async(user_id, params) jid = MergeWorker.perform_async(id, user_id, params.to_h) update_column(:merge_jid, jid) + + # merge_ongoing? depends on merge_jid + # expire etag cache since the attribute is changed without triggering callbacks + expire_etag_cache end # Set off a rebase asynchronously, atomically updating the `rebase_jid` of @@ -408,6 +403,10 @@ class MergeRequest < ApplicationRecord update_column(:rebase_jid, jid) end + + # rebase_in_progress? depends on rebase_jid + # expire etag cache since the attribute is changed without triggering callbacks + expire_etag_cache end def merge_participants @@ -752,7 +751,7 @@ class MergeRequest < ApplicationRecord end def check_mergeability - MergeRequests::MergeabilityCheckService.new(self).execute + MergeRequests::MergeabilityCheckService.new(self).execute(retry_lease: false) end # rubocop: enable CodeReuse/ServiceClass @@ -1249,15 +1248,8 @@ class MergeRequest < ApplicationRecord end def all_commits - # MySQL doesn't support LIMIT in a subquery. - diffs_relation = if Gitlab::Database.postgresql? - merge_request_diffs.recent - else - merge_request_diffs - end - MergeRequestDiffCommit - .where(merge_request_diff: diffs_relation) + .where(merge_request_diff: merge_request_diffs.recent) .limit(10_000) end @@ -1435,4 +1427,11 @@ class MergeRequest < ApplicationRecord variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', value: source_branch.to_s) end end + + def expire_etag_cache + return unless project.namespace + + key = Gitlab::Routing.url_helpers.cached_widget_project_json_merge_request_path(project, self, format: :json) + Gitlab::EtagCaching::Store.new.touch(key) + end end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index f45bd0e03de..2c9dbf2585c 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -196,6 +196,12 @@ class MergeRequestDiff < ApplicationRecord real_size.presence || raw_diffs.size end + def lines_count + strong_memoize(:lines_count) do + diffs.diff_files.sum(&:line_count) + end + end + def raw_diffs(options = {}) if options[:ignore_whitespace_change] @diffs_no_whitespace ||= compare.diffs(options) diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 37c129e843a..2ad2838111e 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -149,29 +149,10 @@ class Milestone < ApplicationRecord end def self.upcoming_ids(projects, groups) - rel = unscoped - .for_projects_and_groups(projects, groups) - .active.where('milestones.due_date > CURRENT_DATE') - - if Gitlab::Database.postgresql? - rel.order(:project_id, :group_id, :due_date).select('DISTINCT ON (project_id, group_id) id') - else - # We need to use MySQL's NULL-safe comparison operator `<=>` here - # because one of `project_id` or `group_id` is always NULL - join_clause = <<~HEREDOC - LEFT OUTER JOIN milestones earlier_milestones - ON milestones.project_id <=> earlier_milestones.project_id - AND milestones.group_id <=> earlier_milestones.group_id - AND milestones.due_date > earlier_milestones.due_date - AND earlier_milestones.due_date > CURRENT_DATE - AND earlier_milestones.state = 'active' - HEREDOC - - rel - .joins(join_clause) - .where('earlier_milestones.id IS NULL') - .select(:id) - end + unscoped + .for_projects_and_groups(projects, groups) + .active.where('milestones.due_date > CURRENT_DATE') + .order(:project_id, :group_id, :due_date).select('DISTINCT ON (project_id, group_id) id') end def participants diff --git a/app/models/namespace.rb b/app/models/namespace.rb index b8d7348268a..9f9c4288667 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -172,6 +172,13 @@ class Namespace < ApplicationRecord end end + # any ancestor can disable emails for all descendants + def emails_disabled? + strong_memoize(:emails_disabled) do + Feature.enabled?(:emails_disabled, self, default_enabled: true) && self_and_ancestors.where(emails_disabled: true).exists? + end + end + def lfs_enabled? # User namespace will always default to the global setting Gitlab.config.lfs.enabled @@ -332,8 +339,6 @@ class Namespace < ApplicationRecord end def force_share_with_group_lock_on_descendants - return unless Group.supports_nested_objects? - # We can't use `descendants.update_all` since Rails will throw away the WITH # RECURSIVE statement. We also can't use WHERE EXISTS since we can't use # different table aliases, hence we're just using WHERE IN. Since we have a diff --git a/app/models/namespace/aggregation_schedule.rb b/app/models/namespace/aggregation_schedule.rb index 0bef352cf24..61a7eb4b576 100644 --- a/app/models/namespace/aggregation_schedule.rb +++ b/app/models/namespace/aggregation_schedule.rb @@ -6,21 +6,13 @@ class Namespace::AggregationSchedule < ApplicationRecord self.primary_key = :namespace_id - DEFAULT_LEASE_TIMEOUT = 3.hours + DEFAULT_LEASE_TIMEOUT = 1.5.hours.to_i REDIS_SHARED_KEY = 'gitlab:update_namespace_statistics_delay'.freeze belongs_to :namespace after_create :schedule_root_storage_statistics - def self.delay_timeout - redis_timeout = Gitlab::Redis::SharedState.with do |redis| - redis.get(REDIS_SHARED_KEY) - end - - redis_timeout.nil? ? DEFAULT_LEASE_TIMEOUT : redis_timeout.to_i - end - def schedule_root_storage_statistics run_after_commit_or_now do try_obtain_lease do @@ -28,7 +20,7 @@ class Namespace::AggregationSchedule < ApplicationRecord .perform_async(namespace_id) Namespaces::RootStatisticsWorker - .perform_in(self.class.delay_timeout, namespace_id) + .perform_in(DEFAULT_LEASE_TIMEOUT, namespace_id) end end end @@ -37,7 +29,7 @@ class Namespace::AggregationSchedule < ApplicationRecord # Used by ExclusiveLeaseGuard def lease_timeout - self.class.delay_timeout + DEFAULT_LEASE_TIMEOUT end # Used by ExclusiveLeaseGuard diff --git a/app/models/note.rb b/app/models/note.rb index 5c31cff9816..a12d1eb7243 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -27,6 +27,10 @@ class Note < ApplicationRecord def values constants.map {|const| self.const_get(const)} end + + def value?(val) + values.include?(val) + end end end @@ -292,7 +296,7 @@ class Note < ApplicationRecord end def special_role=(role) - raise "Role is undefined, #{role} not found in #{SpecialRole.values}" unless SpecialRole.values.include?(role) + raise "Role is undefined, #{role} not found in #{SpecialRole.values}" unless SpecialRole.value?(role) @special_role = role end diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index a7f73c0f29c..8e44e3d8e17 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -4,6 +4,7 @@ class NotificationRecipient include Gitlab::Utils::StrongMemoize attr_reader :user, :type, :reason + def initialize(user, type, **opts) unless NotificationSetting.levels.key?(type) || type == :subscription raise ArgumentError, "invalid type: #{type.inspect}" @@ -30,6 +31,7 @@ class NotificationRecipient def notifiable? return false unless has_access? + return false if emails_disabled? return false if own_activity? # even users with :disabled notifications receive manual subscriptions @@ -109,6 +111,12 @@ class NotificationRecipient private + # They are disabled if the project or group has disallowed it. + # No need to check the group if there is already a project + def emails_disabled? + @project ? @project.emails_disabled? : @group&.emails_disabled? + end + def read_ability return if @skip_read_ability return @read_ability if instance_variable_defined?(:@read_ability) diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index e6e491634ab..27c122d3559 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -22,7 +22,7 @@ class PagesDomain < ApplicationRecord validate :validate_pages_domain validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? } - validate :validate_intermediates, if: ->(domain) { domain.certificate.present? } + validate :validate_intermediates, if: ->(domain) { domain.certificate.present? && domain.certificate_changed? } attr_encrypted :key, mode: :per_attribute_iv_and_salt, diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index f69f0e2dccb..7ae431eaad7 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -7,6 +7,7 @@ class PersonalAccessToken < ApplicationRecord add_authentication_token_field :token, digest: true REDIS_EXPIRY_TIME = 3.minutes + TOKEN_LENGTH = 20 serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize diff --git a/app/models/project.rb b/app/models/project.rb index 2906aca75fc..10679fb1f85 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -55,6 +55,8 @@ class Project < ApplicationRecord VALID_MIRROR_PORTS = [22, 80, 443].freeze VALID_MIRROR_PROTOCOLS = %w(http https ssh git).freeze + SORTING_PREFERENCE_FIELD = :projects_sort + cache_markdown_field :description, pipeline: :description delegate :feature_available?, :builds_enabled?, :wiki_enabled?, @@ -162,7 +164,6 @@ class Project < ApplicationRecord has_one :bugzilla_service has_one :gitlab_issue_tracker_service, inverse_of: :project has_one :external_wiki_service - has_one :kubernetes_service, inverse_of: :project has_one :prometheus_service, inverse_of: :project has_one :mock_ci_service has_one :mock_deployment_service @@ -214,7 +215,7 @@ class Project < ApplicationRecord as: :source, class_name: 'ProjectMember', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :members_and_requesters, as: :source, class_name: 'ProjectMember' - has_many :deploy_keys_projects + has_many :deploy_keys_projects, inverse_of: :project has_many :deploy_keys, through: :deploy_keys_projects has_many :users_star_projects has_many :starrers, through: :users_star_projects, source: :user @@ -277,13 +278,14 @@ class Project < ApplicationRecord has_many :project_deploy_tokens has_many :deploy_tokens, through: :project_deploy_tokens - has_one :auto_devops, class_name: 'ProjectAutoDevops' + has_one :auto_devops, class_name: 'ProjectAutoDevops', inverse_of: :project, autosave: true has_many :custom_attributes, class_name: 'ProjectCustomAttribute' has_many :project_badges, class_name: 'ProjectBadge' has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :remote_mirrors, inverse_of: :project + has_many :cycle_analytics_stages, class_name: 'Analytics::CycleAnalytics::ProjectStage' accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true @@ -357,6 +359,7 @@ class Project < ApplicationRecord scope :sorted_by_activity, -> { reorder(Arel.sql("GREATEST(COALESCE(last_activity_at, '1970-01-01'), COALESCE(last_repository_updated_at, '1970-01-01')) DESC")) } scope :sorted_by_stars_desc, -> { reorder(star_count: :desc) } scope :sorted_by_stars_asc, -> { reorder(star_count: :asc) } + scope :sorted_by_name_asc_limited, ->(limit) { reorder(name: :asc).limit(limit) } scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) } scope :personal, ->(user) { where(namespace_id: user.namespace_id) } @@ -414,12 +417,6 @@ class Project < ApplicationRecord .where(project_ci_cd_settings: { group_runners_enabled: true }) end - scope :missing_kubernetes_namespace, -> (kubernetes_namespaces) do - subquery = kubernetes_namespaces.select('1').where('clusters_kubernetes_namespaces.project_id = projects.id') - - where('NOT EXISTS (?)', subquery) - end - enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, @@ -441,22 +438,6 @@ class Project < ApplicationRecord without_deleted.find_by_id(id) end - # Paginates a collection using a `WHERE id < ?` condition. - # - # before - A project ID to use for filtering out projects with an equal or - # greater ID. If no ID is given, all projects are included. - # - # limit - The maximum number of rows to include. - def self.paginate_in_descending_order_using_id( - before: nil, - limit: Kaminari.config.default_per_page - ) - relation = order_id_desc.limit(limit) - relation = relation.where('projects.id < ?', before) if before - - relation - end - def self.eager_load_namespace_and_owner includes(namespace: :owner) end @@ -653,6 +634,13 @@ class Project < ApplicationRecord alias_method :ancestors, :ancestors_upto + def emails_disabled? + strong_memoize(:emails_disabled) do + # disabling in the namespace overrides the project setting + Feature.enabled?(:emails_disabled, self, default_enabled: true) && (super || namespace.emails_disabled?) + end + end + def lfs_enabled? return namespace.lfs_enabled? if self[:lfs_enabled].nil? @@ -734,16 +722,27 @@ class Project < ApplicationRecord repository.commits_by(oids: oids) end - # ref can't be HEAD, can only be branch/tag name or SHA - def latest_successful_build_for(job_name, ref = default_branch) - latest_pipeline = ci_pipelines.latest_successful_for(ref) + # ref can't be HEAD, can only be branch/tag name + def latest_successful_build_for_ref(job_name, ref = default_branch) + return unless ref + + latest_pipeline = ci_pipelines.latest_successful_for_ref(ref) + return unless latest_pipeline + + latest_pipeline.builds.latest.with_artifacts_archive.find_by(name: job_name) + end + + def latest_successful_build_for_sha(job_name, sha) + return unless sha + + latest_pipeline = ci_pipelines.latest_successful_for_sha(sha) return unless latest_pipeline latest_pipeline.builds.latest.with_artifacts_archive.find_by(name: job_name) end - def latest_successful_build_for!(job_name, ref = default_branch) - latest_successful_build_for(job_name, ref) || raise(ActiveRecord::RecordNotFound.new("Couldn't find job #{job_name}")) + def latest_successful_build_for_ref!(job_name, ref = default_branch) + latest_successful_build_for_ref(job_name, ref) || raise(ActiveRecord::RecordNotFound.new("Couldn't find job #{job_name}")) end def merge_base_commit(first_commit_id, second_commit_id) @@ -1241,6 +1240,14 @@ class Project < ApplicationRecord end end + def has_active_hooks?(hooks_scope = :push_hooks) + hooks.hooks_for(hooks_scope).any? || SystemHook.hooks_for(hooks_scope).any? + end + + def has_active_services?(hooks_scope = :push_hooks) + services.public_send(hooks_scope).any? # rubocop:disable GitlabSecurity/PublicSend + end + def valid_repo? repository.exists? rescue @@ -1496,12 +1503,19 @@ class Project < ApplicationRecord !namespace.share_with_group_lock end - def pipeline_for(ref, sha = nil) + def pipeline_for(ref, sha = nil, id = nil) sha ||= commit(ref).try(:sha) - return unless sha - ci_pipelines.order(id: :desc).find_by(sha: sha, ref: ref) + if id.present? + pipelines_for(ref, sha).find_by(id: id) + else + pipelines_for(ref, sha).take + end + end + + def pipelines_for(ref, sha) + ci_pipelines.order(id: :desc).where(sha: sha, ref: ref) end def latest_successful_pipeline_for_default_branch @@ -1510,12 +1524,12 @@ class Project < ApplicationRecord end @latest_successful_pipeline_for_default_branch = - ci_pipelines.latest_successful_for(default_branch) + ci_pipelines.latest_successful_for_ref(default_branch) end def latest_successful_pipeline_for(ref = nil) if ref && ref != default_branch - ci_pipelines.latest_successful_for(ref) + ci_pipelines.latest_successful_for_ref(ref) else latest_successful_pipeline_for_default_branch end @@ -1831,11 +1845,16 @@ class Project < ApplicationRecord end def ci_variables_for(ref:, environment: nil) - # EE would use the environment - if protected_for?(ref) - variables + result = if protected_for?(ref) + variables + else + variables.unprotected + end + + if environment + result.on_environment(environment) else - variables.unprotected + result.where(environment_scope: '*') end end @@ -1858,8 +1877,12 @@ class Project < ApplicationRecord end end - def deployment_variables(environment: nil) - deployment_platform(environment: environment)&.predefined_variables(project: self) || [] + def deployment_variables(environment:) + platform = deployment_platform(environment: environment) + + return [] unless platform.present? + + platform.predefined_variables(project: self, environment_name: environment) end def auto_devops_variables @@ -1869,16 +1892,24 @@ class Project < ApplicationRecord end def append_or_update_attribute(name, value) - old_values = public_send(name.to_s) # rubocop:disable GitlabSecurity/PublicSend + if Project.reflect_on_association(name).try(:macro) == :has_many + # if this is 1-to-N relation, update the parent object + value.each do |item| + item.update!( + Project.reflect_on_association(name).foreign_key => id) + end + + # force to drop relation cache + public_send(name).reset # rubocop:disable GitlabSecurity/PublicSend - if Project.reflect_on_association(name).try(:macro) == :has_many && old_values.any? - update_attribute(name, old_values + value) + # succeeded + true else + # if this is another relation or attribute, update just object update_attribute(name, value) end - - rescue ActiveRecord::RecordNotSaved => e - handle_update_attribute_error(e, value) + rescue ActiveRecord::RecordInvalid => e + raise e, "Failed to set #{name}: #{e.message}" end # Tries to set repository as read_only, checking for existing Git transfers in progress beforehand @@ -2267,18 +2298,6 @@ class Project < ApplicationRecord ContainerRepository.build_root_repository(self).has_tags? end - def handle_update_attribute_error(ex, value) - if ex.message.start_with?('Failed to replace') - if value.respond_to?(:each) - invalid = value.detect(&:invalid?) - - raise ex, ([ex.message] + invalid.errors.full_messages).join(' ') if invalid - end - end - - raise ex - end - def fetch_branch_allows_collaboration(user, branch_name = nil) return false unless user diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index 67c12363a3c..e11d0c48b4b 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true class ProjectAutoDevops < ApplicationRecord - include IgnorableColumn - - ignore_column :domain - - belongs_to :project + belongs_to :project, inverse_of: :auto_devops enum deploy_strategy: { continuous: 0, diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 62aec4351db..a3793d9937b 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -1,24 +1,47 @@ # frozen_string_literal: true +require 'slack-notifier' module ChatMessage class PipelineMessage < BaseMessage + MAX_VISIBLE_JOBS = 10 + + attr_reader :user attr_reader :ref_type attr_reader :ref attr_reader :status + attr_reader :detailed_status attr_reader :duration + attr_reader :finished_at attr_reader :pipeline_id + attr_reader :failed_stages + attr_reader :failed_jobs + + attr_reader :project + attr_reader :commit + attr_reader :committer + attr_reader :pipeline def initialize(data) super + @user = data[:user] @user_name = data.dig(:user, :username) || 'API' pipeline_attributes = data[:object_attributes] @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' @ref = pipeline_attributes[:ref] @status = pipeline_attributes[:status] + @detailed_status = pipeline_attributes[:detailed_status] @duration = pipeline_attributes[:duration].to_i + @finished_at = pipeline_attributes[:finished_at] ? Time.parse(pipeline_attributes[:finished_at]).to_i : nil @pipeline_id = pipeline_attributes[:id] + @failed_jobs = Array(data[:builds]).select { |b| b[:status] == 'failed' }.reverse # Show failed jobs from oldest to newest + @failed_stages = @failed_jobs.map { |j| j[:stage] }.uniq + + @project = Project.find(data[:project][:id]) + @commit = project.commit_by(oid: data[:commit][:id]) + @committer = commit.committer + @pipeline = Ci::Pipeline.find(pipeline_id) end def pretext @@ -28,38 +51,145 @@ module ChatMessage def attachments return message if markdown - [{ text: format(message), color: attachment_color }] + return [{ text: format(message), color: attachment_color }] unless fancy_notifications? + + [{ + fallback: format(message), + color: attachment_color, + author_name: user_combined_name, + author_icon: user_avatar, + author_link: author_url, + title: s_("ChatMessage|Pipeline #%{pipeline_id} %{humanized_status} in %{duration}") % + { + pipeline_id: pipeline_id, + humanized_status: humanized_status, + duration: pretty_duration(duration) + }, + title_link: pipeline_url, + fields: attachments_fields, + footer: project.name, + footer_icon: project.avatar_url(only_path: false), + ts: finished_at + }] end def activity { - title: "Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status}", - subtitle: "in #{project_link}", - text: "in #{pretty_duration(duration)}", + title: s_("ChatMessage|Pipeline %{pipeline_link} of %{ref_type} %{branch_link} by %{user_combined_name} %{humanized_status}") % + { + pipeline_link: pipeline_link, + ref_type: ref_type, + branch_link: branch_link, + user_combined_name: user_combined_name, + humanized_status: humanized_status + }, + subtitle: s_("ChatMessage|in %{project_link}") % { project_link: project_link }, + text: s_("ChatMessage|in %{duration}") % { duration: pretty_duration(duration) }, image: user_avatar || '' } end private + def fancy_notifications? + Feature.enabled?(:fancy_pipeline_slack_notifications, default_enabled: true) + end + + def failed_stages_field + { + title: s_("ChatMessage|Failed stage").pluralize(failed_stages.length), + value: Slack::Notifier::LinkFormatter.format(failed_stages_links), + short: true + } + end + + def failed_jobs_field + { + title: s_("ChatMessage|Failed job").pluralize(failed_jobs.length), + value: Slack::Notifier::LinkFormatter.format(failed_jobs_links), + short: true + } + end + + def yaml_error_field + { + title: s_("ChatMessage|Invalid CI config YAML file"), + value: pipeline.yaml_errors, + short: false + } + end + + def attachments_fields + fields = [ + { + title: ref_type == "tag" ? s_("ChatMessage|Tag") : s_("ChatMessage|Branch"), + value: Slack::Notifier::LinkFormatter.format(ref_name_link), + short: true + }, + { + title: s_("ChatMessage|Commit"), + value: Slack::Notifier::LinkFormatter.format(commit_link), + short: true + } + ] + + fields << failed_stages_field if failed_stages.any? + fields << failed_jobs_field if failed_jobs.any? + fields << yaml_error_field if pipeline.has_yaml_errors? + + fields + end + def message - "#{project_link}: Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status} in #{pretty_duration(duration)}" + s_("ChatMessage|%{project_link}: Pipeline %{pipeline_link} of %{ref_type} %{branch_link} by %{user_combined_name} %{humanized_status} in %{duration}") % + { + project_link: project_link, + pipeline_link: pipeline_link, + ref_type: ref_type, + branch_link: branch_link, + user_combined_name: user_combined_name, + humanized_status: humanized_status, + duration: pretty_duration(duration) + } end def humanized_status - case status - when 'success' - 'passed' + if fancy_notifications? + case status + when 'success' + detailed_status == "passed with warnings" ? s_("ChatMessage|has passed with warnings") : s_("ChatMessage|has passed") + when 'failed' + s_("ChatMessage|has failed") + else + status + end else - status + case status + when 'success' + s_("ChatMessage|passed") + when 'failed' + s_("ChatMessage|failed") + else + status + end end end def attachment_color - if status == 'success' - 'good' + if fancy_notifications? + case status + when 'success' + detailed_status == 'passed with warnings' ? 'warning' : 'good' + else + 'danger' + end else - 'danger' + case status + when 'success' + 'good' + else + 'danger' + end end end @@ -71,16 +201,83 @@ module ChatMessage "[#{ref}](#{branch_url})" end + def project_url + project.web_url + end + def project_link - "[#{project_name}](#{project_url})" + "[#{project.name}](#{project_url})" + end + + def pipeline_failed_jobs_url + "#{project_url}/pipelines/#{pipeline_id}/failures" end def pipeline_url - "#{project_url}/pipelines/#{pipeline_id}" + if fancy_notifications? && failed_jobs.any? + pipeline_failed_jobs_url + else + "#{project_url}/pipelines/#{pipeline_id}" + end end def pipeline_link "[##{pipeline_id}](#{pipeline_url})" end + + def job_url(job) + "#{project_url}/-/jobs/#{job[:id]}" + end + + def job_link(job) + "[#{job[:name]}](#{job_url(job)})" + end + + def failed_jobs_links + failed = failed_jobs.slice(0, MAX_VISIBLE_JOBS) + truncated = failed_jobs.slice(MAX_VISIBLE_JOBS, failed_jobs.size) + + failed_links = failed.map { |job| job_link(job) } + + unless truncated.blank? + failed_links << s_("ChatMessage|and [%{count} more](%{pipeline_failed_jobs_url})") % { + count: truncated.size, + pipeline_failed_jobs_url: pipeline_failed_jobs_url + } + end + + failed_links.join(I18n.translate(:'support.array.words_connector')) + end + + def stage_link(stage) + # All stages link to the pipeline page + "[#{stage}](#{pipeline_url})" + end + + def failed_stages_links + failed_stages.map { |s| stage_link(s) }.join(I18n.translate(:'support.array.words_connector')) + end + + def commit_url + Gitlab::UrlBuilder.build(commit) + end + + def commit_link + "[#{commit.title}](#{commit_url})" + end + + def commits_page_url + "#{project_url}/commits/#{ref}" + end + + def ref_name_link + "[#{ref}](#{commits_page_url})" + end + + def author_url + return unless user && committer + + Gitlab::UrlBuilder.build(committer) + end end end diff --git a/app/models/project_services/emails_on_push_service.rb b/app/models/project_services/emails_on_push_service.rb index 45de64a9990..8ca40138a8f 100644 --- a/app/models/project_services/emails_on_push_service.rb +++ b/app/models/project_services/emails_on_push_service.rb @@ -24,6 +24,7 @@ class EmailsOnPushService < Service def execute(push_data) return unless supported_events.include?(push_data[:object_kind]) + return if project.emails_disabled? EmailsOnPushWorker.perform_async( project_id, diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index e571700fd02..d08fcd8954d 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -31,7 +31,7 @@ class JiraService < IssueTrackerService # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 def self.reference_pattern(only_long: true) - @reference_pattern ||= /(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)/ + @reference_pattern ||= /(?<issue>\b#{Gitlab::Regex.jira_issue_key_regex})/ end def initialize_properties @@ -54,7 +54,7 @@ class JiraService < IssueTrackerService username: self.username, password: self.password, site: URI.join(url, '/').to_s, # Intended to find the root - context_path: url.path.chomp('/'), + context_path: url.path, auth_type: :basic, read_timeout: 120, use_cookies: true, @@ -103,6 +103,12 @@ class JiraService < IssueTrackerService "#{url}/secure/CreateIssue.jspa" end + alias_method :original_url, :url + + def url + original_url&.chomp('/') + end + def execute(push) # This method is a no-op, because currently JiraService does not # support any events. diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb deleted file mode 100644 index 9f5c226f4c9..00000000000 --- a/app/models/project_services/kubernetes_service.rb +++ /dev/null @@ -1,133 +0,0 @@ -# frozen_string_literal: true - -class KubernetesService < Service - default_value_for :category, 'deployment' - - # Namespace defaults to the project path, but can be overridden in case that - # is an invalid or inappropriate name - prop_accessor :namespace - - # Access to kubernetes is directly through the API - prop_accessor :api_url - - # Bearer authentication - # TODO: user/password auth, client certificates - prop_accessor :token - - # Provide a custom CA bundle for self-signed deployments - prop_accessor :ca_pem - - with_options presence: true, if: :activated? do - validates :api_url, public_url: true - validates :token - end - - before_validation :enforce_namespace_to_lower_case - - attr_accessor :skip_deprecation_validation - - validate :deprecation_validation, unless: :skip_deprecation_validation - - validates :namespace, - allow_blank: true, - length: 1..63, - if: :activated?, - format: { - with: Gitlab::Regex.kubernetes_namespace_regex, - message: Gitlab::Regex.kubernetes_namespace_regex_message - } - - def self.supported_events - %w() - end - - def can_test? - false - end - - def initialize_properties - self.properties = {} if properties.nil? - end - - def title - 'Kubernetes' - end - - def description - 'Kubernetes / OpenShift integration' - end - - def self.to_param - 'kubernetes' - end - - def fields - [ - { type: 'text', - name: 'api_url', - title: 'API URL', - placeholder: 'Kubernetes API URL, like https://kube.example.com/' }, - { type: 'textarea', - name: 'ca_pem', - title: 'CA Certificate', - placeholder: 'Certificate Authority bundle (PEM format)' }, - { type: 'text', - name: 'namespace', - title: 'Project namespace (optional/unique)', - placeholder: namespace_placeholder }, - { type: 'text', - name: 'token', - title: 'Token', - placeholder: 'Service token' } - ] - end - - def deprecated? - true - end - - def editable? - false - end - - def deprecation_message - content = if project - _("Kubernetes service integration has been disabled. Fields on this page are not used by GitLab, you can configure your Kubernetes clusters using the new <a href=\"%{url}\"/>Kubernetes Clusters</a> page") % { - url: Gitlab::Routing.url_helpers.project_clusters_path(project) - } - else - _("The instance-level Kubernetes service integration is disabled. Your data has been migrated to an <a href=\"%{url}\"/>instance-level cluster</a>.") % { - url: Gitlab::Routing.url_helpers.admin_clusters_path - } - end - - content.html_safe - end - - TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze - - private - - def namespace_placeholder - default_namespace || TEMPLATE_PLACEHOLDER - end - - def default_namespace - return unless project - - slug = "#{project.path}-#{project.id}".downcase - slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') - end - - def enforce_namespace_to_lower_case - self.namespace = self.namespace&.downcase - end - - def deprecation_validation - return if active_changed?(from: true, to: false) || (new_record? && !active?) - - if deprecated? - errors[:base] << deprecation_message - end - end -end diff --git a/app/models/project_services/mock_deployment_service.rb b/app/models/project_services/mock_deployment_service.rb index 1103cb11e73..6f2b0f7747f 100644 --- a/app/models/project_services/mock_deployment_service.rb +++ b/app/models/project_services/mock_deployment_service.rb @@ -24,7 +24,7 @@ class MockDeploymentService < Service %w() end - def predefined_variables(project:) + def predefined_variables(project:, environment_name:) [] end diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index c68a9d923c8..6eff2ea2e3a 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -63,15 +63,16 @@ class PrometheusService < MonitoringService # Check we can connect to the Prometheus API def test(*args) - Gitlab::PrometheusClient.new(prometheus_client).ping - + prometheus_client.ping { success: true, result: 'Checked API endpoint' } rescue Gitlab::PrometheusClient::Error => err { success: false, result: err } end def prometheus_client - RestClient::Resource.new(api_url, max_redirects: 0) if should_return_client? + return unless should_return_client? + + Gitlab::PrometheusClient.new(api_url) end def prometheus_available? @@ -84,7 +85,7 @@ class PrometheusService < MonitoringService private def should_return_client? - api_url && manual_configuration? && active? && valid? + api_url.present? && manual_configuration? && active? && valid? end def synchronize_service_state diff --git a/app/models/project_services/slash_commands_service.rb b/app/models/project_services/slash_commands_service.rb index 5f5cff97808..cb16ad75d14 100644 --- a/app/models/project_services/slash_commands_service.rb +++ b/app/models/project_services/slash_commands_service.rb @@ -35,6 +35,8 @@ class SlashCommandsService < Service chat_user = find_chat_user(params) if chat_user&.user + return Gitlab::SlashCommands::Presenters::Access.new.access_denied unless chat_user.user.can?(:use_slash_commands) + Gitlab::SlashCommands::Command.new(project, chat_user, params).execute else url = authorize_chat_name_url(params) diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 3802d258664..47999a3694e 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -93,13 +93,7 @@ class ProjectStatistics < ApplicationRecord def schedule_namespace_aggregation_worker run_after_commit do - next unless schedule_aggregation_worker? - Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id) end end - - def schedule_aggregation_worker? - Feature.enabled?(:update_statistics_namespace, project&.root_ancestor) - end end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index c91add6439f..4a19e05bf76 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -85,6 +85,10 @@ class ProjectWiki list_pages(limit: 1).empty? end + def exists? + !empty? + end + # Lists wiki pages of the repository. # # limit - max number of pages returned by the method. diff --git a/app/models/prometheus_metric.rb b/app/models/prometheus_metric.rb index b8e7673dcf5..c7786500c5c 100644 --- a/app/models/prometheus_metric.rb +++ b/app/models/prometheus_metric.rb @@ -32,6 +32,10 @@ class PrometheusMetric < ApplicationRecord Gitlab::Prometheus::Metric.new(id: id, title: title, required_metrics: required_metrics, weight: 0, y_label: y_label, queries: queries) end + def to_metric_hash + queries.first.merge(metric_id: id) + end + def queries [ { diff --git a/app/models/prometheus_metric_enums.rb b/app/models/prometheus_metric_enums.rb index 6cb22cc69cd..d58f825f222 100644 --- a/app/models/prometheus_metric_enums.rb +++ b/app/models/prometheus_metric_enums.rb @@ -9,13 +9,17 @@ module PrometheusMetricEnums aws_elb: -3, nginx: -4, kubernetes: -5, - nginx_ingress: -6, + nginx_ingress: -6 + }.merge(custom_groups).freeze + end - # custom/user groups + # custom/user groups + def self.custom_groups + { business: 0, response: 1, system: 2 - } + }.freeze end def self.group_details @@ -50,16 +54,20 @@ module PrometheusMetricEnums group_title: _('System metrics (Kubernetes)'), required_metrics: %w(container_memory_usage_bytes container_cpu_usage_seconds_total), priority: 5 - }.freeze, + }.freeze + }.merge(custom_group_details).freeze + end - # custom/user groups + # custom/user groups + def self.custom_group_details + { business: { group_title: _('Business metrics (Custom)'), priority: 0 }.freeze, response: { group_title: _('Response metrics (Custom)'), - priority: -5 + priority: -5 }.freeze, system: { group_title: _('System metrics (Custom)'), diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb index 2e4769364c6..22f60802257 100644 --- a/app/models/redirect_route.rb +++ b/app/models/redirect_route.rb @@ -11,11 +11,7 @@ class RedirectRoute < ApplicationRecord uniqueness: { case_sensitive: false } scope :matching_path_and_descendants, -> (path) do - wheres = if Gitlab::Database.postgresql? - 'LOWER(redirect_routes.path) = LOWER(?) OR LOWER(redirect_routes.path) LIKE LOWER(?)' - else - 'redirect_routes.path = ? OR redirect_routes.path LIKE ?' - end + wheres = 'LOWER(redirect_routes.path) = LOWER(?) OR LOWER(redirect_routes.path) LIKE LOWER(?)' where(wheres, path, "#{sanitize_sql_like(path)}/%") end diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index af705b29f7a..c9ee0653d86 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -4,6 +4,8 @@ class RemoteMirror < ApplicationRecord include AfterCommitQueue include MirrorAuthentication + MAX_FIRST_RUNTIME = 3.hours + MAX_INCREMENTAL_RUNTIME = 1.hour PROTECTED_BACKOFF_DELAY = 1.minute UNPROTECTED_BACKOFF_DELAY = 5.minutes @@ -31,11 +33,18 @@ class RemoteMirror < ApplicationRecord scope :enabled, -> { where(enabled: true) } scope :started, -> { with_update_status(:started) } - scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + + scope :stuck, -> do + started + .where('(last_update_started_at < ? AND last_update_at IS NOT NULL)', + MAX_INCREMENTAL_RUNTIME.ago) + .or(where('(last_update_started_at < ? AND last_update_at IS NULL)', + MAX_FIRST_RUNTIME.ago)) + end state_machine :update_status, initial: :none do event :update_start do - transition [:none, :finished, :failed] => :started + transition any => :started end event :update_finish do @@ -46,9 +55,14 @@ class RemoteMirror < ApplicationRecord transition started: :failed end + event :update_retry do + transition started: :to_retry + end + state :started state :finished state :failed + state :to_retry after_transition any => :started do |remote_mirror, _| Gitlab::Metrics.add_event(:remote_mirrors_running) @@ -138,16 +152,27 @@ class RemoteMirror < ApplicationRecord end def updated_since?(timestamp) - last_update_started_at && last_update_started_at > timestamp && !update_failed? + return false if failed? + + last_update_started_at && last_update_started_at > timestamp end def mark_for_delete_if_blank_url mark_for_destruction if url.blank? end - def mark_as_failed(error_message) - update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message)) - update_fail + def update_error_message(error_message) + self.last_error = Gitlab::UrlSanitizer.sanitize(error_message) + end + + def mark_for_retry!(error_message) + update_error_message(error_message) + update_retry! + end + + def mark_as_failed!(error_message) + update_error_message(error_message) + update_fail! end def url=(value) @@ -173,7 +198,7 @@ class RemoteMirror < ApplicationRecord result = URI.parse(url) result.password = '*****' if result.password - result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.user = '*****' if result.user && result.user != 'git' # tokens or other data may be saved as user result.to_s end @@ -190,6 +215,18 @@ class RemoteMirror < ApplicationRecord update_column(:error_notification_sent, true) end + def backoff_delay + if self.only_protected_branches + PROTECTED_BACKOFF_DELAY + else + UNPROTECTED_BACKOFF_DELAY + end + end + + def max_runtime + last_update_at.present? ? MAX_INCREMENTAL_RUNTIME : MAX_FIRST_RUNTIME + end + private def store_credentials @@ -219,14 +256,6 @@ class RemoteMirror < ApplicationRecord self.last_update_started_at >= Time.now - backoff_delay end - def backoff_delay - if self.only_protected_branches - PROTECTED_BACKOFF_DELAY - else - UNPROTECTED_BACKOFF_DELAY - end - end - def reset_fields update_columns( last_error: nil, diff --git a/app/models/repository.rb b/app/models/repository.rb index 187382ad182..b957b9b0bdd 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -7,6 +7,9 @@ class Repository REF_KEEP_AROUND = 'keep-around'.freeze REF_ENVIRONMENTS = 'environments'.freeze + ARCHIVE_CACHE_TIME = 60 # Cache archives referred to by a (mutable) ref for 1 minute + ARCHIVE_CACHE_TIME_IMMUTABLE = 3600 # Cache archives referred to by an immutable reference for 1 hour + RESERVED_REFS_NAMES = %W[ heads tags @@ -236,13 +239,13 @@ class Repository def branch_exists?(branch_name) return false unless raw_repository - branch_names.include?(branch_name) + branch_names_include?(branch_name) end def tag_exists?(tag_name) return false unless raw_repository - tag_names.include?(tag_name) + tag_names_include?(tag_name) end def ref_exists?(ref) @@ -386,11 +389,15 @@ class Repository expire_statistics_caches end - # Runs code after a repository has been created. - def after_create + def expire_status_cache expire_exists_cache expire_root_ref_cache expire_emptiness_caches + end + + # Runs code after a repository has been created. + def after_create + expire_status_cache repository_event(:create_repository) end @@ -415,25 +422,29 @@ class Repository end # Runs code before pushing (= creating or removing) a tag. + # + # Note that this doesn't expire the tags. You may need to call + # expire_caches_for_tags or expire_tags_cache. def before_push_tag + repository_event(:push_tag) + end + + def expire_caches_for_tags expire_statistics_caches expire_emptiness_caches expire_tags_cache - - repository_event(:push_tag) end # Runs code before removing a tag. def before_remove_tag - expire_tags_cache - expire_statistics_caches + expire_caches_for_tags repository_event(:remove_tag) end # Runs code after removing a tag. def after_remove_tag - expire_tags_cache + expire_caches_for_tags end # Runs code after the HEAD of a repository is changed. @@ -457,8 +468,8 @@ class Repository end # Runs code after a new branch has been created. - def after_create_branch - expire_branches_cache + def after_create_branch(expire_cache: true) + expire_branches_cache if expire_cache repository_event(:push_branch) end @@ -471,8 +482,8 @@ class Repository end # Runs code after an existing branch has been removed. - def after_remove_branch - expire_branches_cache + def after_remove_branch(expire_cache: true) + expire_branches_cache if expire_cache end def method_missing(msg, *args, &block) @@ -554,10 +565,10 @@ class Repository end delegate :branch_names, to: :raw_repository - cache_method :branch_names, fallback: [] + cache_method_as_redis_set :branch_names, fallback: [] delegate :tag_names, to: :raw_repository - cache_method :tag_names, fallback: [] + cache_method_as_redis_set :tag_names, fallback: [] delegate :branch_count, :tag_count, :has_visible_content?, to: :raw_repository cache_method :branch_count, fallback: 0 @@ -1119,6 +1130,10 @@ class Repository @cache ||= Gitlab::RepositoryCache.new(self) end + def redis_set_cache + @redis_set_cache ||= Gitlab::RepositorySetCache.new(self) + end + def request_store_cache @request_store_cache ||= Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore) end diff --git a/app/models/service.rb b/app/models/service.rb index 752467622f2..f6d8fb1fb46 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -260,7 +260,6 @@ class Service < ApplicationRecord hipchat irker jira - kubernetes mattermost_slash_commands mattermost packagist diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 55da37c9545..9a2640db9ca 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -16,7 +16,7 @@ class SystemNoteMetadata < ApplicationRecord commit description merge confidential visible label assignee cross_reference title time_tracking branch milestone discussion task moved opened closed merged duplicate locked unlocked - outdated tag due_date + outdated tag due_date pinned_embed ].freeze validates :note, presence: true diff --git a/app/models/user.rb b/app/models/user.rb index 0fd3daa3383..6131a8dc710 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -282,6 +282,17 @@ class User < ApplicationRecord scope :for_todos, -> (todos) { where(id: todos.select(:user_id)) } scope :with_emails, -> { preload(:emails) } scope :with_dashboard, -> (dashboard) { where(dashboard: dashboard) } + scope :with_public_profile, -> { where(private_profile: false) } + + def self.with_visible_profile(user) + return with_public_profile if user.nil? + + if user.admin? + all + else + with_public_profile.or(where(id: user.id)) + end + end # Limits the users to those that have TODOs, optionally in the given state. # @@ -427,18 +438,20 @@ class User < ApplicationRecord order = <<~SQL CASE - WHEN users.name = %{query} THEN 0 - WHEN users.username = %{query} THEN 1 - WHEN users.email = %{query} THEN 2 + WHEN users.name = :query THEN 0 + WHEN users.username = :query THEN 1 + WHEN users.email = :query THEN 2 ELSE 3 END SQL + sanitized_order_sql = Arel.sql(sanitize_sql_array([order, query: query])) + where( fuzzy_arel_match(:name, query, lower_exact_match: true) .or(fuzzy_arel_match(:username, query, lower_exact_match: true)) .or(arel_table[:email].eq(query)) - ).reorder(order % { query: ApplicationRecord.connection.quote(query) }, :name) + ).reorder(sanitized_order_sql, :name) end # Limits the result set to users _not_ in the given query/list of IDs. @@ -933,7 +946,7 @@ class User < ApplicationRecord end def project_deploy_keys - DeployKey.unscoped.in_projects(authorized_projects.pluck(:id)).distinct(:id) + DeployKey.in_projects(authorized_projects.select(:id)).distinct(:id) end def highest_role @@ -941,11 +954,10 @@ class User < ApplicationRecord end def accessible_deploy_keys - @accessible_deploy_keys ||= begin - key_ids = project_deploy_keys.pluck(:id) - key_ids.push(*DeployKey.are_public.pluck(:id)) - DeployKey.where(id: key_ids) - end + DeployKey.from_union([ + DeployKey.where(id: project_deploy_keys.select(:deploy_key_id)), + DeployKey.are_public + ]) end def created_by @@ -1259,6 +1271,11 @@ class User < ApplicationRecord end end + def notification_email_for(notification_group) + # Return group-specific email address if present, otherwise return global notification email address + notification_group&.notification_email_for(self) || notification_email + end + def notification_settings_for(source) if notification_settings.loaded? notification_settings.find do |notification| @@ -1490,6 +1507,13 @@ class User < ApplicationRecord super end + # override from Devise::Confirmable + def confirmation_period_valid? + return false if Feature.disabled?(:soft_email_confirmation) + + super + end + private def default_private_profile_to_false diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index f1326f4c8cb..b236250c24e 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -26,7 +26,7 @@ class UserPreference < ApplicationRecord def set_notes_filter(filter_id, issuable) # No need to update the column if the value is already set. - if filter_id && NOTES_FILTERS.values.include?(filter_id) + if filter_id && NOTES_FILTERS.value?(filter_id) field = notes_filter_field_for(issuable) self[field] = filter_id diff --git a/app/models/users_star_project.rb b/app/models/users_star_project.rb index 9be6bd2e6f3..c633e2d8b3d 100644 --- a/app/models/users_star_project.rb +++ b/app/models/users_star_project.rb @@ -1,10 +1,38 @@ # frozen_string_literal: true class UsersStarProject < ApplicationRecord + include Sortable + belongs_to :project, counter_cache: :star_count, touch: true belongs_to :user validates :user, presence: true validates :user_id, uniqueness: { scope: [:project_id] } validates :project, presence: true + + alias_attribute :starred_since, :created_at + + scope :order_user_name_asc, -> { joins(:user).merge(User.order_name_asc) } + scope :order_user_name_desc, -> { joins(:user).merge(User.order_name_desc) } + scope :by_project, -> (project) { where(project_id: project.id) } + scope :with_visible_profile, -> (user) { joins(:user).merge(User.with_visible_profile(user)) } + scope :with_public_profile, -> { joins(:user).merge(User.with_public_profile) } + scope :preload_users, -> { preload(:user) } + + class << self + def sort_by_attribute(method) + order_method = method || 'id_desc' + + case order_method.to_s + when 'name_asc' then order_user_name_asc + when 'name_desc' then order_user_name_desc + else + order_by(order_method) + end + end + + def search(query) + joins(:user).merge(User.search(query)) + end + end end |