diff options
Diffstat (limited to 'app/models/concerns')
| -rw-r--r-- | app/models/concerns/analytics/cycle_analytics/stage_event_model.rb | 40 | ||||
| -rw-r--r-- | app/models/concerns/ci/contextable.rb | 47 | ||||
| -rw-r--r-- | app/models/concerns/ci/has_variable.rb | 17 | ||||
| -rw-r--r-- | app/models/concerns/cross_database_modification.rb | 122 | ||||
| -rw-r--r-- | app/models/concerns/has_environment_scope.rb | 8 | ||||
| -rw-r--r-- | app/models/concerns/issuable.rb | 14 | ||||
| -rw-r--r-- | app/models/concerns/mirror_authentication.rb | 9 | ||||
| -rw-r--r-- | app/models/concerns/noteable.rb | 6 | ||||
| -rw-r--r-- | app/models/concerns/packages/debian/distribution.rb | 8 | ||||
| -rw-r--r-- | app/models/concerns/resolvable_discussion.rb | 10 | ||||
| -rw-r--r-- | app/models/concerns/taskable.rb | 15 | ||||
| -rw-r--r-- | app/models/concerns/timebox.rb | 13 | ||||
| -rw-r--r-- | app/models/concerns/token_authenticatable.rb | 12 | ||||
| -rw-r--r-- | app/models/concerns/token_authenticatable_strategies/base.rb | 36 |
14 files changed, 269 insertions, 88 deletions
diff --git a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb index 324e0fb57cb..7cc4bc569d3 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true - module Analytics module CycleAnalytics module StageEventModel @@ -16,12 +15,39 @@ module Analytics scope :authored, ->(user) { where(author_id: user) } scope :with_milestone_id, ->(milestone_id) { where(milestone_id: milestone_id) } scope :end_event_is_not_happened_yet, -> { where(end_event_timestamp: nil) } + scope :order_by_end_event, -> (direction) do + # ORDER BY end_event_timestamp, merge_request_id/issue_id, start_event_timestamp + # start_event_timestamp must be included in the ORDER BY clause for the duration + # calculation to work: SELECT end_event_timestamp - start_event_timestamp + keyset_order( + :end_event_timestamp => { order_expression: arel_order(arel_table[:end_event_timestamp], direction), distinct: false }, + issuable_id_column => { order_expression: arel_order(arel_table[issuable_id_column], direction), distinct: true }, + :start_event_timestamp => { order_expression: arel_order(arel_table[:start_event_timestamp], direction), distinct: false } + ) + end + scope :order_by_duration, -> (direction) do + # ORDER BY EXTRACT('epoch', end_event_timestamp - start_event_timestamp) + duration = Arel::Nodes::Subtraction.new( + arel_table[:end_event_timestamp], + arel_table[:start_event_timestamp] + ) + duration_in_seconds = Arel::Nodes::Extract.new(duration, :epoch) + + keyset_order( + :total_time => { order_expression: arel_order(duration_in_seconds, direction), distinct: false, sql_type: 'double precision' }, + issuable_id_column => { order_expression: arel_order(arel_table[issuable_id_column], direction), distinct: true } + ) + end end def issuable_id attributes[self.class.issuable_id_column.to_s] end + def total_time + read_attribute(:total_time) || (end_event_timestamp - start_event_timestamp).to_f + end + class_methods do def upsert_data(data) upsert_values = data.map do |row| @@ -68,6 +94,18 @@ module Analytics result = connection.execute(query) result.cmd_tuples end + + def keyset_order(column_definition_options) + built_definitions = column_definition_options.map do |attribute_name, column_options| + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(attribute_name: attribute_name, **column_options) + end + + order(Gitlab::Pagination::Keyset::Order.build(built_definitions)) + end + + def arel_order(arel_node, direction) + direction.to_sym == :desc ? arel_node.desc : arel_node.asc + end end end end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index ed3b422251f..88b7bb89b89 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -11,26 +11,9 @@ module Ci # def scoped_variables(environment: expanded_environment_name, dependencies: true) track_duration do - variables = pipeline.variables_builder.scoped_variables(self, environment: environment, dependencies: dependencies) - - next variables if pipeline.use_variables_builder_definitions? - - variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) - variables.concat(runner.predefined_variables) if runnable? && runner - variables.concat(kubernetes_variables) - variables.concat(deployment_variables(environment: environment)) - variables.concat(yaml_variables) - variables.concat(user_variables) - variables.concat(dependency_variables) if dependencies - variables.concat(secret_instance_variables) - variables.concat(secret_group_variables(environment: environment)) - variables.concat(secret_project_variables(environment: environment)) - variables.concat(trigger_request.user_variables) if trigger_request - variables.concat(pipeline.variables) - variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule - - variables + pipeline + .variables_builder + .scoped_variables(self, environment: environment, dependencies: dependencies) end end @@ -60,29 +43,5 @@ module Ci scoped_variables(environment: nil, dependencies: false) end end - - def user_variables - pipeline.variables_builder.user_variables(user) - end - - def kubernetes_variables - pipeline.variables_builder.kubernetes_variables(self) - end - - def deployment_variables(environment:) - pipeline.variables_builder.deployment_variables(job: self, environment: environment) - end - - def secret_instance_variables - pipeline.variables_builder.secret_instance_variables(ref: git_ref) - end - - def secret_group_variables(environment: expanded_environment_name) - pipeline.variables_builder.secret_group_variables(environment: environment, ref: git_ref) - end - - def secret_project_variables(environment: expanded_environment_name) - pipeline.variables_builder.secret_project_variables(environment: environment, ref: git_ref) - end end end diff --git a/app/models/concerns/ci/has_variable.rb b/app/models/concerns/ci/has_variable.rb index 7309469c77e..3b437fbba16 100644 --- a/app/models/concerns/ci/has_variable.rb +++ b/app/models/concerns/ci/has_variable.rb @@ -31,7 +31,24 @@ module Ci end def to_runner_variable + var_cache_key = to_runner_variable_cache_key + + return uncached_runner_variable unless var_cache_key + + ::Gitlab::SafeRequestStore.fetch(var_cache_key) { uncached_runner_variable } + end + + private + + def uncached_runner_variable { key: key, value: value, public: false, file: file? } end + + def to_runner_variable_cache_key + return unless persisted? + + variable_id = read_attribute(self.class.primary_key) + "#{self.class}#to_runner_variable:#{variable_id}:#{key}" + end end end diff --git a/app/models/concerns/cross_database_modification.rb b/app/models/concerns/cross_database_modification.rb new file mode 100644 index 00000000000..85645e482f6 --- /dev/null +++ b/app/models/concerns/cross_database_modification.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module CrossDatabaseModification + extend ActiveSupport::Concern + + class TransactionStackTrackRecord + DEBUG_STACK = Rails.env.test? && ENV['DEBUG_GITLAB_TRANSACTION_STACK'] + LOG_FILENAME = Rails.root.join("log", "gitlab_transaction_stack.log") + + EXCLUDE_DEBUG_TRACE = %w[ + lib/gitlab/database/query_analyzer + app/models/concerns/cross_database_modification.rb + ].freeze + + def self.logger + @logger ||= Logger.new(LOG_FILENAME, formatter: ->(_, _, _, msg) { Gitlab::Json.dump(msg) + "\n" }) + end + + def self.log_gitlab_transactions_stack(action: nil, example: nil) + return unless DEBUG_STACK + + message = "gitlab_transactions_stack performing #{action}" + message += " in example #{example}" if example + + cleaned_backtrace = Gitlab::BacktraceCleaner.clean_backtrace(caller) + .reject { |line| EXCLUDE_DEBUG_TRACE.any? { |exclusion| line.include?(exclusion) } } + .first(5) + + logger.warn({ + message: message, + action: action, + gitlab_transactions_stack: ::ApplicationRecord.gitlab_transactions_stack, + caller: cleaned_backtrace, + thread: Thread.current.object_id + }) + end + + def initialize(subject, gitlab_schema) + @subject = subject + @gitlab_schema = gitlab_schema + @subject.gitlab_transactions_stack.push(gitlab_schema) + + self.class.log_gitlab_transactions_stack(action: :after_push) + end + + def done! + unless @done + @done = true + + self.class.log_gitlab_transactions_stack(action: :before_pop) + @subject.gitlab_transactions_stack.pop + end + + true + end + + def trigger_transactional_callbacks? + false + end + + def before_committed! + end + + def rolledback!(force_restore_state: false, should_run_callbacks: true) + done! + end + + def committed!(should_run_callbacks: true) + done! + end + end + + included do + private_class_method :gitlab_schema + end + + class_methods do + def gitlab_transactions_stack + Thread.current[:gitlab_transactions_stack] ||= [] + end + + def transaction(**options, &block) + if track_gitlab_schema_in_current_transaction? + super(**options) do + # Hook into current transaction to ensure that once + # the `COMMIT` is executed the `gitlab_transactions_stack` + # will be allowing to execute `after_commit_queue` + record = TransactionStackTrackRecord.new(self, gitlab_schema) + + begin + connection.current_transaction.add_record(record) + + yield + ensure + record.done! + end + end + else + super(**options, &block) + end + end + + def track_gitlab_schema_in_current_transaction? + return false unless Feature::FlipperFeature.table_exists? + + Feature.enabled?(:track_gitlab_schema_in_current_transaction, default_enabled: :yaml) + rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad + false + end + + def gitlab_schema + case self.name + when 'ActiveRecord::Base', 'ApplicationRecord' + :gitlab_main + when 'Ci::ApplicationRecord' + :gitlab_ci + else + Gitlab::Database::GitlabSchema.table_schema(table_name) if table_name + end + end + end +end diff --git a/app/models/concerns/has_environment_scope.rb b/app/models/concerns/has_environment_scope.rb index 9553abe4dd3..c01996c0c4c 100644 --- a/app/models/concerns/has_environment_scope.rb +++ b/app/models/concerns/has_environment_scope.rb @@ -70,6 +70,14 @@ module HasEnvironmentScope relation end + + scope :for_environment, ->(environment) do + if environment + on_environment(environment) + else + where(environment_scope: '*') + end + end end def environment_scope=(new_environment_scope) diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index dcd80201d3f..0138c0ad20f 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -194,6 +194,8 @@ module Issuable end def supports_escalation? + return false unless ::Feature.enabled?(:incident_escalations, project) + incident? end @@ -363,9 +365,10 @@ module Issuable end # Includes table keys in group by clause when sorting - # preventing errors in postgres + # preventing errors in Postgres + # + # Returns an array of Arel columns # - # Returns an array of arel columns def grouping_columns(sort) sort = sort.to_s grouping_columns = [arel_table[:id]] @@ -384,9 +387,10 @@ module Issuable end # Includes all table keys in group by clause when sorting - # preventing errors in postgres when using CTE search optimisation + # preventing errors in Postgres when using CTE search optimization + # + # Returns an array of Arel columns # - # Returns an array of arel columns def issue_grouping_columns(use_cte: false) if use_cte attribute_names.map { |attr| arel_table[attr.to_sym] } @@ -576,7 +580,7 @@ module Issuable ## # Overridden in MergeRequest # - def wipless_title_changed(old_title) + def draftless_title_changed(old_title) old_title != title end end diff --git a/app/models/concerns/mirror_authentication.rb b/app/models/concerns/mirror_authentication.rb index 4dbf4dcec77..14c8be93ce0 100644 --- a/app/models/concerns/mirror_authentication.rb +++ b/app/models/concerns/mirror_authentication.rb @@ -4,11 +4,6 @@ # implements support for persisting the necessary data in a `credentials` # serialized attribute. It also needs an `url` method to be defined module MirrorAuthentication - SSH_PRIVATE_KEY_OPTS = { - type: 'RSA', - bits: 4096 - }.freeze - extend ActiveSupport::Concern included do @@ -84,10 +79,10 @@ module MirrorAuthentication return if ssh_private_key.blank? comment = "git@#{::Gitlab.config.gitlab.host}" - ::SSHKey.new(ssh_private_key, comment: comment).ssh_public_key + SSHData::PrivateKey.parse(ssh_private_key).first.public_key.openssh(comment: comment) end def generate_ssh_private_key! - self.ssh_private_key = ::SSHKey.generate(SSH_PRIVATE_KEY_OPTS).private_key + self.ssh_private_key = SSHData::PrivateKey::RSA.generate(4096).openssl.to_pem end end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index ea4fe5b27dc..c1aac235d33 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -176,7 +176,7 @@ module Noteable Gitlab::Routing.url_helpers.project_noteable_notes_path( project, - target_type: self.class.name.underscore, + target_type: noteable_target_type_name, target_id: id ) end @@ -201,6 +201,10 @@ module Noteable project_email.sub('@', "-#{iid}@") end + def noteable_target_type_name + model_name.singular + end + private # Synthetic system notes don't have discussion IDs because these are generated dynamically diff --git a/app/models/concerns/packages/debian/distribution.rb b/app/models/concerns/packages/debian/distribution.rb index 2d46889ce6a..1520ec0828e 100644 --- a/app/models/concerns/packages/debian/distribution.rb +++ b/app/models/concerns/packages/debian/distribution.rb @@ -97,12 +97,8 @@ module Packages end def package_files - if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) - ::Packages::PackageFile.installable - .for_package_ids(packages.select(:id)) - else - ::Packages::PackageFile.for_package_ids(packages.select(:id)) - end + ::Packages::PackageFile.installable + .for_package_ids(packages.select(:id)) end private diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index aae338e9759..92a88d2f7c8 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -99,6 +99,12 @@ module ResolvableDiscussion update { |notes| notes.unresolve! } end + def clear_memoized_values + self.class.memoized_values.each do |name| + clear_memoization(name) + end + end + private def update @@ -110,8 +116,6 @@ module ResolvableDiscussion # Set the notes array to the updated notes @notes = notes_relation.fresh.to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables - self.class.memoized_values.each do |name| - clear_memoization(name) - end + clear_memoized_values end end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index 4d1c1d44af7..e41a0ca28f9 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -15,17 +15,16 @@ module Taskable INCOMPLETE_PATTERN = /(\[\s\])/.freeze ITEM_PATTERN = %r{ ^ - (?:(?:>\s{0,4})*) # optional blockquote characters - (?:\s*(?:[-+*]|(?:\d+\.)))+ # list prefix (one or more) required - task item has to be always in a list - \s+ # whitespace prefix has to be always presented for a list item - (\[\s\]|\[[xX]\]) # checkbox - (\s.+) # followed by whitespace and some text. + (?:(?:>\s{0,4})*) # optional blockquote characters + ((?:\s*(?:[-+*]|(?:\d+\.)))+) # list prefix (one or more) required - task item has to be always in a list + \s+ # whitespace prefix has to be always presented for a list item + (\[\s\]|\[[xX]\]) # checkbox + (\s.+) # followed by whitespace and some text. }x.freeze def self.get_tasks(content) - content.to_s.scan(ITEM_PATTERN).map do |checkbox, label| - # ITEM_PATTERN strips out the hyphen, but Item requires it. Rabble rabble. - TaskList::Item.new("- #{checkbox}", label.strip) + content.to_s.scan(ITEM_PATTERN).map do |prefix, checkbox, label| + TaskList::Item.new("#{prefix} #{checkbox}", label.strip) end end diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb index 3fe9d7f4d71..943ef3fa59f 100644 --- a/app/models/concerns/timebox.rb +++ b/app/models/concerns/timebox.rb @@ -51,7 +51,7 @@ module Timebox validate :dates_within_4_digits cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description + cache_markdown_field :description, issuable_reference_expansion_enabled: true belongs_to :project belongs_to :group @@ -125,17 +125,6 @@ module Timebox fuzzy_search(query, [:title, :description]) end - # Searches for timeboxes with a matching title. - # - # This method uses ILIKE on PostgreSQL - # - # query - The search query as a String - # - # Returns an ActiveRecord::Relation. - def search_title(query) - fuzzy_search(query, [:title]) - end - def filter_by_state(timeboxes, state) case state when 'closed' then timeboxes.closed diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 34c8630bb90..f44ad8ebe90 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -64,6 +64,18 @@ module TokenAuthenticatable mod.define_method("format_#{token_field}") do |token| token end + + mod.define_method("#{token_field}_expires_at") do + strategy.expires_at(self) + end + + mod.define_method("#{token_field}_expired?") do + strategy.expired?(self) + end + + mod.define_method("#{token_field}_with_expiration") do + strategy.token_with_expiration(self) + end end def token_authenticatable_module diff --git a/app/models/concerns/token_authenticatable_strategies/base.rb b/app/models/concerns/token_authenticatable_strategies/base.rb index f72a41f06b1..2cec4ab460e 100644 --- a/app/models/concerns/token_authenticatable_strategies/base.rb +++ b/app/models/concerns/token_authenticatable_strategies/base.rb @@ -7,6 +7,7 @@ module TokenAuthenticatableStrategies def initialize(klass, token_field, options) @klass = klass @token_field = token_field + @expires_at_field = "#{token_field}_expires_at" @options = options end @@ -44,6 +45,25 @@ module TokenAuthenticatableStrategies instance.save! if Gitlab::Database.read_write? end + def expires_at(instance) + instance.read_attribute(@expires_at_field) + end + + def expired?(instance) + return false unless expirable? && token_expiration_enforced? + + exp = expires_at(instance) + !!exp && Time.current > exp + end + + def expirable? + !!@options[:expires_at] + end + + def token_with_expiration(instance) + API::Support::TokenWithExpiration.new(self, instance) + end + def self.fabricate(model, field, options) if options[:digest] && options[:encrypted] raise ArgumentError, _('Incompatible options set!') @@ -64,6 +84,10 @@ module TokenAuthenticatableStrategies new_token = generate_available_token formatted_token = format_token(instance, new_token) set_token(instance, formatted_token) + + if expirable? + instance[@expires_at_field] = @options[:expires_at].to_proc.call(instance) + end end def unique @@ -82,11 +106,21 @@ module TokenAuthenticatableStrategies end def relation(unscoped) - unscoped ? @klass.unscoped : @klass + unscoped ? @klass.unscoped : @klass.where(not_expired) end def token_set?(instance) raise NotImplementedError end + + def token_expiration_enforced? + return true unless @options[:expiration_enforced?] + + @options[:expiration_enforced?].to_proc.call(@klass) + end + + def not_expired + Arel.sql("#{@expires_at_field} IS NULL OR #{@expires_at_field} >= NOW()") if expirable? && token_expiration_enforced? + end end end |