diff options
Diffstat (limited to 'lib')
62 files changed, 767 insertions, 1027 deletions
diff --git a/lib/api/container_registry.rb b/lib/api/container_registry.rb index e4493910196..7d9b5e1a598 100644 --- a/lib/api/container_registry.rb +++ b/lib/api/container_registry.rb @@ -115,12 +115,8 @@ module API authorize! :read_container_image, repository end - def authorize_update_container_image! - authorize! :update_container_image, repository - end - def authorize_destroy_container_image! - authorize! :admin_container_image, repository + authorize! :destroy_container_image, repository end def authorize_admin_container_image! diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index df8f0470063..7f8300a0c2f 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,27 +1,41 @@ # frozen_string_literal: true require 'asciidoctor' -require 'asciidoctor/converter/html5' -require "asciidoctor-plantuml" +require 'asciidoctor-plantuml' +require 'asciidoctor/extensions' +require 'gitlab/asciidoc/html5_converter' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - DEFAULT_ADOC_ATTRS = [ - 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', - 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font', - 'outfilesuffix=.adoc' - ].freeze + MAX_INCLUDE_DEPTH = 5 + DEFAULT_ADOC_ATTRS = { + 'showtitle' => true, + 'idprefix' => 'user-content-', + 'idseparator' => '-', + 'env' => 'gitlab', + 'env-gitlab' => '', + 'source-highlighter' => 'html-pipeline', + 'icons' => 'font', + 'outfilesuffix' => '.adoc', + 'max-include-depth' => MAX_INCLUDE_DEPTH + }.freeze # Public: Converts the provided Asciidoc markup into HTML. # # input - the source text in Asciidoc format + # context - :commit, :project, :ref, :requested_path # def self.render(input, context) + extensions = proc do + include_processor ::Gitlab::Asciidoc::IncludeProcessor.new(context) + end + asciidoc_opts = { safe: :secure, backend: :gitlab_html5, - attributes: DEFAULT_ADOC_ATTRS } + attributes: DEFAULT_ADOC_ATTRS, + extensions: extensions } context[:pipeline] = :ascii_doc @@ -40,29 +54,5 @@ module Gitlab conf.txt_enable = false end end - - class Html5Converter < Asciidoctor::Converter::Html5Converter - extend Asciidoctor::Converter::Config - - register_for 'gitlab_html5' - - def stem(node) - return super unless node.style.to_sym == :latexmath - - %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>) - end - - def inline_quoted(node) - return super unless node.type.to_sym == :latexmath - - %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>) - end - - private - - def id_attribute(node) - node.id ? %( id="#{node.id}") : nil - end - end end end diff --git a/lib/gitlab/asciidoc/html5_converter.rb b/lib/gitlab/asciidoc/html5_converter.rb new file mode 100644 index 00000000000..2c5c74e4789 --- /dev/null +++ b/lib/gitlab/asciidoc/html5_converter.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'asciidoctor' +require 'asciidoctor/converter/html5' + +module Gitlab + module Asciidoc + class Html5Converter < Asciidoctor::Converter::Html5Converter + extend Asciidoctor::Converter::Config + + register_for 'gitlab_html5' + + def stem(node) + return super unless node.style.to_sym == :latexmath + + %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>) + end + + def inline_quoted(node) + return super unless node.type.to_sym == :latexmath + + %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>) + end + + private + + def id_attribute(node) + node.id ? %( id="#{node.id}") : nil + end + end + end +end diff --git a/lib/gitlab/asciidoc/include_processor.rb b/lib/gitlab/asciidoc/include_processor.rb new file mode 100644 index 00000000000..c6fbf540e9c --- /dev/null +++ b/lib/gitlab/asciidoc/include_processor.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require 'asciidoctor/include_ext/include_processor' + +module Gitlab + module Asciidoc + # Asciidoctor extension for processing includes (macro include::[]) within + # documents inside the same repository. + class IncludeProcessor < Asciidoctor::IncludeExt::IncludeProcessor + extend ::Gitlab::Utils::Override + + def initialize(context) + super(logger: Gitlab::AppLogger) + + @context = context + @repository = context[:project].try(:repository) + + # Note: Asciidoctor calls #freeze on extensions, so we can't set new + # instance variables after initialization. + @cache = { + uri_types: {} + } + end + + protected + + override :include_allowed? + def include_allowed?(target, reader) + doc = reader.document + + return false if doc.attributes.fetch('max-include-depth').to_i < 1 + return false if target_uri?(target) + + true + end + + override :resolve_target_path + def resolve_target_path(target, reader) + return unless repository.try(:exists?) + + base_path = reader.include_stack.empty? ? requested_path : reader.file + path = resolve_relative_path(target, base_path) + + path if Gitlab::Git::Blob.find(repository, ref, path) + end + + override :read_lines + def read_lines(filename, selector) + blob = read_blob(ref, filename) + + if selector + blob.data.each_line.select.with_index(1, &selector) + else + blob.data + end + end + + override :unresolved_include! + def unresolved_include!(target, reader) + reader.unshift_line("*[ERROR: include::#{target}[] - unresolved directive]*") + end + + private + + attr_accessor :context, :repository, :cache + + # Gets a Blob at a path for a specific revision. + # This method will check that the Blob exists and contains readable text. + # + # revision - The String SHA1. + # path - The String file path. + # + # Returns a Blob + def read_blob(ref, filename) + blob = repository&.blob_at(ref, filename) + + raise 'Blob not found' unless blob + raise 'File is not readable' unless blob.readable_text? + + blob + end + + # Resolves the given relative path of file in repository into canonical + # path based on the specified base_path. + # + # Examples: + # + # # File in the same directory as the current path + # resolve_relative_path("users.adoc", "doc/api/README.adoc") + # # => "doc/api/users.adoc" + # + # # File in the same directory, which is also the current path + # resolve_relative_path("users.adoc", "doc/api") + # # => "doc/api/users.adoc" + # + # # Going up one level to a different directory + # resolve_relative_path("../update/7.14-to-8.0.adoc", "doc/api/README.adoc") + # # => "doc/update/7.14-to-8.0.adoc" + # + # Returns a String + def resolve_relative_path(path, base_path) + p = Pathname(base_path) + p = p.dirname unless p.extname.empty? + p += path + + p.cleanpath.to_s + end + + def current_commit + cache[:current_commit] ||= context[:commit] || repository&.commit(ref) + end + + def ref + context[:ref] || context[:project].default_branch + end + + def requested_path + cache[:requested_path] ||= Addressable::URI.unescape(context[:requested_path]) + end + + def uri_type(path) + cache[:uri_types][path] ||= current_commit&.uri_type(path) + end + end + end +end diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb deleted file mode 100644 index da8265a3a5f..00000000000 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys - class GpgKey < ActiveRecord::Base - self.table_name = 'gpg_keys' - - include EachBatch - include ShaAttribute - - sha_attribute :primary_keyid - sha_attribute :fingerprint - - has_many :subkeys, class_name: 'GpgKeySubkey' - end - - class GpgKeySubkey < ActiveRecord::Base - self.table_name = 'gpg_key_subkeys' - - include ShaAttribute - - sha_attribute :keyid - sha_attribute :fingerprint - end - - def perform(gpg_key_id) - gpg_key = GpgKey.find_by(id: gpg_key_id) - - return if gpg_key.nil? - return if gpg_key.subkeys.any? - - create_subkeys(gpg_key) - update_signatures(gpg_key) - end - - private - - def create_subkeys(gpg_key) - gpg_subkeys = Gitlab::Gpg.subkeys_from_key(gpg_key.key) - - gpg_subkeys[gpg_key.primary_keyid.upcase]&.each do |subkey_data| - gpg_key.subkeys.build(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint]) - end - - # Improve latency by doing all INSERTs in a single call - GpgKey.transaction do - gpg_key.save! - end - end - - def update_signatures(gpg_key) - return unless gpg_key.subkeys.exists? - - InvalidGpgSignatureUpdateWorker.perform_async(gpg_key.id) - end -end diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb deleted file mode 100644 index 664ead1af44..00000000000 --- a/lib/gitlab/background_migration/delete_diff_files.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class DeleteDiffFiles - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - - belongs_to :merge_request - has_many :merge_request_diff_files - end - - class MergeRequestDiffFile < ActiveRecord::Base - self.table_name = 'merge_request_diff_files' - end - - DEAD_TUPLES_THRESHOLD = 50_000 - VACUUM_WAIT_TIME = 5.minutes - - def perform(ids) - @ids = ids - - # We should reschedule until deadtuples get in a desirable - # state (e.g. < 50_000). That may take more than one reschedule. - # - if should_wait_deadtuple_vacuum? - reschedule - return - end - - prune_diff_files - end - - def should_wait_deadtuple_vacuum? - return false unless Gitlab::Database.postgresql? - - diff_files_dead_tuples_count >= DEAD_TUPLES_THRESHOLD - end - - private - - def reschedule - BackgroundMigrationWorker.perform_in(VACUUM_WAIT_TIME, self.class.name.demodulize, [@ids]) - end - - def diffs_collection - MergeRequestDiff.where(id: @ids) - end - - def diff_files_dead_tuples_count - dead_tuple = - execute_statement("SELECT n_dead_tup FROM pg_stat_all_tables "\ - "WHERE relname = 'merge_request_diff_files'")[0] - - dead_tuple&.fetch('n_dead_tup', 0).to_i - end - - def prune_diff_files - removed = 0 - updated = 0 - - MergeRequestDiff.transaction do - updated = diffs_collection.update_all(state: 'without_files') - removed = MergeRequestDiffFile.where(merge_request_diff_id: @ids).delete_all - end - - log_info("Removed #{removed} merge_request_diff_files rows, "\ - "updated #{updated} merge_request_diffs rows") - end - - def execute_statement(sql) - ActiveRecord::Base.connection.execute(sql) - end - - def log_info(message) - Rails.logger.info("BackgroundMigration::DeleteDiffFiles - #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb deleted file mode 100644 index 58df74cfa9b..00000000000 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/AbcSize -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class DeserializeMergeRequestDiffsAndCommits - attr_reader :diff_ids, :commit_rows, :file_rows - - class Error < StandardError - def backtrace - cause.backtrace - end - end - - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - end - - BUFFER_ROWS = 1000 - DIFF_FILE_BUFFER_ROWS = 100 - - def perform(start_id, stop_id) - merge_request_diffs = MergeRequestDiff - .select(:id, :st_commits, :st_diffs) - .where('st_commits IS NOT NULL OR st_diffs IS NOT NULL') - .where(id: start_id..stop_id) - - reset_buffers! - - merge_request_diffs.each do |merge_request_diff| - commits, files = single_diff_rows(merge_request_diff) - - diff_ids << merge_request_diff.id - commit_rows.concat(commits) - file_rows.concat(files) - - if diff_ids.length > BUFFER_ROWS || - commit_rows.length > BUFFER_ROWS || - file_rows.length > DIFF_FILE_BUFFER_ROWS - - flush_buffers! - end - end - - flush_buffers! - rescue => e - Rails.logger.info("#{self.class.name}: failed for IDs #{merge_request_diffs.map(&:id)} with #{e.class.name}") - - raise Error.new(e.inspect) - end - - private - - def reset_buffers! - @diff_ids = [] - @commit_rows = [] - @file_rows = [] - end - - def flush_buffers! - if diff_ids.any? - commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice| - bulk_insert('merge_request_diff_commits', commit_rows_slice) - end - - file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice| - bulk_insert('merge_request_diff_files', file_rows_slice) - end - - MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) - end - - reset_buffers! - end - - def bulk_insert(table, rows) - Gitlab::Database.bulk_insert(table, rows) - rescue ActiveRecord::RecordNotUnique - ids = rows.map { |row| row[:merge_request_diff_id] }.uniq.sort - - Rails.logger.info("#{self.class.name}: rows inserted twice for IDs #{ids}") - end - - def single_diff_rows(merge_request_diff) - sha_attribute = Gitlab::Database::ShaAttribute.new - commits = YAML.load(merge_request_diff.st_commits) rescue [] - commits ||= [] - - commit_rows = commits.map.with_index do |commit, index| - commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids) - sha = commit_hash.delete(:id) - - commit_hash.merge( - merge_request_diff_id: merge_request_diff.id, - relative_order: index, - sha: sha_attribute.serialize(sha) - ) - end - - diffs = YAML.load(merge_request_diff.st_diffs) rescue [] - diffs = [] unless valid_raw_diffs?(diffs) - - file_rows = diffs.map.with_index do |diff, index| - diff_hash = diff.to_hash.with_indifferent_access.merge( - binary: false, - merge_request_diff_id: merge_request_diff.id, - relative_order: index - ) - - diff_hash.tap do |hash| - diff_text = hash[:diff] - - hash[:too_large] = !!hash[:too_large] - - hash[:a_mode] ||= guess_mode(hash[:new_file], hash[:diff]) - hash[:b_mode] ||= guess_mode(hash[:deleted_file], hash[:diff]) - - # Compatibility with old diffs created with Psych. - if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? - hash[:binary] = true - hash[:diff] = [diff_text].pack('m0') - end - end - end - - [commit_rows, file_rows] - end - - # This doesn't have to be 100% accurate, because it's only used for - # display - it won't change file modes in the repository. Submodules are - # created as 600, regular files as 644. - def guess_mode(file_missing, diff) - return '0' if file_missing - - diff.include?('Subproject commit') ? '160000' : '100644' - end - - # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as - # valid, because we don't render them usefully anyway. - def valid_raw_diffs?(diffs) - return false unless diffs.respond_to?(:each) - - diffs.all? { |diff| diff.is_a?(Hash) } - end - end - end -end diff --git a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb new file mode 100644 index 00000000000..e948cedaad5 --- /dev/null +++ b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MergeRequestAssigneesMigrationProgressCheck + include Gitlab::Utils::StrongMemoize + + RESCHEDULE_DELAY = 3.hours + WORKER = 'PopulateMergeRequestAssigneesTable'.freeze + DeadJobsError = Class.new(StandardError) + + def perform + raise DeadJobsError, "Only dead background jobs in the queue for #{WORKER}" if !ongoing? && dead_jobs? + + if ongoing? + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name) + else + Feature.enable(:multiple_merge_request_assignees) + end + end + + private + + def dead_jobs? + strong_memoize(:dead_jobs) do + migration_klass.dead_jobs?(WORKER) + end + end + + def ongoing? + strong_memoize(:ongoing) do + migration_klass.exists?(WORKER) || migration_klass.retrying_jobs?(WORKER) + end + end + + def migration_klass + Gitlab::BackgroundMigration + end + end + # rubocop: enable Style/Documentation + end +end diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb deleted file mode 100644 index 036fe641757..00000000000 --- a/lib/gitlab/background_migration/populate_external_pipeline_source.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateExternalPipelineSource - module Migratable - class Pipeline < ActiveRecord::Base - self.table_name = 'ci_pipelines' - - def self.sources - { - unknown: nil, - push: 1, - web: 2, - trigger: 3, - schedule: 4, - api: 5, - external: 6 - } - end - end - - class CommitStatus < ActiveRecord::Base - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - - scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') } - scope :of_type, -> (type) { where('type=?', type) } - end - end - - def perform(start_id, stop_id) - external_pipelines(start_id, stop_id) - .update_all(source: Migratable::Pipeline.sources[:external]) - end - - private - - def external_pipelines(start_id, stop_id) - Migratable::Pipeline.where(id: (start_id..stop_id)) - .where( - 'EXISTS (?) AND NOT EXISTS (?)', - Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1), - Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1) - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_import_state.rb b/lib/gitlab/background_migration/populate_import_state.rb deleted file mode 100644 index 695a2a713c5..00000000000 --- a/lib/gitlab/background_migration/populate_import_state.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration creates all the records on the - # import state table for projects that are considered imports or forks - class PopulateImportState - def perform(start_id, end_id) - move_attributes_data_to_import_state(start_id, end_id) - rescue ActiveRecord::RecordNotUnique - retry - end - - def move_attributes_data_to_import_state(start_id, end_id) - Rails.logger.info("#{self.class.name} - Moving import attributes data to project mirror data table: #{start_id} - #{end_id}") - - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO project_mirror_data (project_id, status, jid, last_error) - SELECT id, import_status, import_jid, import_error - FROM projects - WHERE projects.import_status != 'none' - AND projects.id BETWEEN #{start_id} AND #{end_id} - AND NOT EXISTS ( - SELECT id - FROM project_mirror_data - WHERE project_id = projects.id - ) - SQL - - ActiveRecord::Base.connection.execute <<~SQL - UPDATE projects - SET import_status = 'none' - WHERE import_status != 'none' - AND id BETWEEN #{start_id} AND #{end_id} - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb deleted file mode 100644 index d89ce358bb9..00000000000 --- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateMergeRequestMetricsWithEventsData - def perform(min_merge_request_id, max_merge_request_id) - insert_metrics_for_range(min_merge_request_id, max_merge_request_id) - update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) - end - - # Inserts merge_request_metrics records for merge_requests without it for - # a given merge request batch. - def insert_metrics_for_range(min, max) - metrics_not_exists_clause = - <<-SQL.strip_heredoc - NOT EXISTS (SELECT 1 FROM merge_request_metrics - WHERE merge_request_metrics.merge_request_id = merge_requests.id) - SQL - - MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| - select_sql = batch.select(:id, :created_at, :updated_at).to_sql - - execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") - end - end - - def update_metrics_with_events_data(min, max) - if Gitlab::Database.postgresql? - # Uses WITH syntax in order to update merged and closed events with a single UPDATE. - # WITH is not supported by MySQL. - update_events_for_range(min, max) - else - update_merged_events_for_range(min, max) - update_closed_events_for_range(min, max) - end - end - - private - - # Updates merge_request_metrics latest_closed_at, latest_closed_by_id and merged_by_id - # based on the latest event records on events table for a given merge request batch. - def update_events_for_range(min, max) - sql = <<-SQL.strip_heredoc - WITH events_for_update AS ( - SELECT DISTINCT ON (target_id, action) target_id, action, author_id, updated_at - FROM events - WHERE target_id BETWEEN #{min} AND #{max} - AND target_type = 'MergeRequest' - AND action IN (#{Event::CLOSED},#{Event::MERGED}) - ORDER BY target_id, action, id DESC - ) - UPDATE merge_request_metrics met - SET latest_closed_at = latest_closed.updated_at, - latest_closed_by_id = latest_closed.author_id, - merged_by_id = latest_merged.author_id - FROM (SELECT * FROM events_for_update WHERE action = #{Event::CLOSED}) AS latest_closed - FULL OUTER JOIN - (SELECT * FROM events_for_update WHERE action = #{Event::MERGED}) AS latest_merged - USING (target_id) - WHERE target_id = merge_request_id; - SQL - - execute(sql) - end - - # Updates merge_request_metrics latest_closed_at, latest_closed_by_id based on the latest closed - # records on events table for a given merge request batch. - def update_closed_events_for_range(min, max) - sql = - <<-SQL.strip_heredoc - UPDATE merge_request_metrics metrics, - (#{select_events(min, max, Event::CLOSED)}) closed_events - SET metrics.latest_closed_by_id = closed_events.author_id, - metrics.latest_closed_at = closed_events.updated_at #{where_matches_closed_events}; - SQL - - execute(sql) - end - - # Updates merge_request_metrics merged_by_id based on the latest merged - # records on events table for a given merge request batch. - def update_merged_events_for_range(min, max) - sql = - <<-SQL.strip_heredoc - UPDATE merge_request_metrics metrics, - (#{select_events(min, max, Event::MERGED)}) merged_events - SET metrics.merged_by_id = merged_events.author_id #{where_matches_merged_events}; - SQL - - execute(sql) - end - - def execute(sql) - @connection ||= ActiveRecord::Base.connection - @connection.execute(sql) - end - - def select_events(min, max, action) - select_max_event_id = <<-SQL.strip_heredoc - SELECT max(id) - FROM events - WHERE action = #{action} - AND target_type = 'MergeRequest' - AND target_id BETWEEN #{min} AND #{max} - GROUP BY target_id - SQL - - <<-SQL.strip_heredoc - SELECT author_id, updated_at, target_id - FROM events - WHERE id IN(#{select_max_event_id}) - SQL - end - - def where_matches_closed_events - <<-SQL.strip_heredoc - WHERE metrics.merge_request_id = closed_events.target_id - AND metrics.latest_closed_at IS NULL - AND metrics.latest_closed_by_id IS NULL - SQL - end - - def where_matches_merged_events - <<-SQL.strip_heredoc - WHERE metrics.merge_request_id = merged_events.target_id - AND metrics.merged_by_id IS NULL - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb deleted file mode 100644 index 37592d67dd9..00000000000 --- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateMergeRequestMetricsWithEventsDataImproved - CLOSED_EVENT_ACTION = 3 - MERGED_EVENT_ACTION = 7 - - def perform(min_merge_request_id, max_merge_request_id) - insert_metrics_for_range(min_merge_request_id, max_merge_request_id) - update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) - end - - # Inserts merge_request_metrics records for merge_requests without it for - # a given merge request batch. - def insert_metrics_for_range(min, max) - metrics_not_exists_clause = - <<-SQL.strip_heredoc - NOT EXISTS (SELECT 1 FROM merge_request_metrics - WHERE merge_request_metrics.merge_request_id = merge_requests.id) - SQL - - MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| - select_sql = batch.select(:id, :created_at, :updated_at).to_sql - - execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") - end - end - - def update_metrics_with_events_data(min, max) - if Gitlab::Database.postgresql? - psql_update_metrics_with_events_data(min, max) - else - mysql_update_metrics_with_events_data(min, max) - end - end - - def psql_update_metrics_with_events_data(min, max) - update_sql = <<-SQL.strip_heredoc - UPDATE merge_request_metrics - SET (latest_closed_at, - latest_closed_by_id) = - ( SELECT updated_at, - author_id - FROM events - WHERE target_id = merge_request_id - AND target_type = 'MergeRequest' - AND action = #{CLOSED_EVENT_ACTION} - ORDER BY id DESC - LIMIT 1 ), - merged_by_id = - ( SELECT author_id - FROM events - WHERE target_id = merge_request_id - AND target_type = 'MergeRequest' - AND action = #{MERGED_EVENT_ACTION} - ORDER BY id DESC - LIMIT 1 ) - WHERE merge_request_id BETWEEN #{min} AND #{max} - SQL - - execute(update_sql) - end - - def mysql_update_metrics_with_events_data(min, max) - closed_updated_at_subquery = mysql_events_select(:updated_at, CLOSED_EVENT_ACTION) - closed_author_id_subquery = mysql_events_select(:author_id, CLOSED_EVENT_ACTION) - merged_author_id_subquery = mysql_events_select(:author_id, MERGED_EVENT_ACTION) - - update_sql = <<-SQL.strip_heredoc - UPDATE merge_request_metrics - SET latest_closed_at = (#{closed_updated_at_subquery}), - latest_closed_by_id = (#{closed_author_id_subquery}), - merged_by_id = (#{merged_author_id_subquery}) - WHERE merge_request_id BETWEEN #{min} AND #{max} - SQL - - execute(update_sql) - end - - def mysql_events_select(column, action) - <<-SQL.strip_heredoc - SELECT #{column} FROM events - WHERE target_id = merge_request_id - AND target_type = 'MergeRequest' - AND action = #{action} - ORDER BY id DESC - LIMIT 1 - SQL - end - - def execute(sql) - @connection ||= ActiveRecord::Base.connection - @connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/redact_links.rb b/lib/gitlab/background_migration/redact_links.rb deleted file mode 100644 index 92256e59a6c..00000000000 --- a/lib/gitlab/background_migration/redact_links.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -require_relative 'redact_links/redactable' - -module Gitlab - module BackgroundMigration - class RedactLinks - class Note < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'notes' - self.inheritance_column = :_type_disabled - end - - class Issue < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'issues' - self.inheritance_column = :_type_disabled - end - - class MergeRequest < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'merge_requests' - self.inheritance_column = :_type_disabled - end - - class Snippet < ActiveRecord::Base - include EachBatch - include ::Gitlab::BackgroundMigration::RedactLinks::Redactable - - self.table_name = 'snippets' - self.inheritance_column = :_type_disabled - end - - def perform(model_name, field, start_id, stop_id) - link_pattern = "%/sent_notifications/" + ("_" * 32) + "/unsubscribe%" - model = "Gitlab::BackgroundMigration::RedactLinks::#{model_name}".constantize - - model.where("#{field} like ?", link_pattern).where(id: start_id..stop_id).each do |resource| - resource.redact_field!(field) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/redact_links/redactable.rb b/lib/gitlab/background_migration/redact_links/redactable.rb deleted file mode 100644 index baab34221f1..00000000000 --- a/lib/gitlab/background_migration/redact_links/redactable.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class RedactLinks - module Redactable - extend ActiveSupport::Concern - - def redact_field!(field) - self[field].gsub!(%r{/sent_notifications/\h{32}/unsubscribe}, '/sent_notifications/REDACTED/unsubscribe') - - if self.changed? - self.update_columns(field => self[field], - "#{field}_html" => nil) - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/rollback_import_state_data.rb b/lib/gitlab/background_migration/rollback_import_state_data.rb deleted file mode 100644 index a7c986747d8..00000000000 --- a/lib/gitlab/background_migration/rollback_import_state_data.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration migrates all the data of import_state - # back to the projects table for projects that are considered imports or forks - class RollbackImportStateData - def perform(start_id, end_id) - move_attributes_data_to_project(start_id, end_id) - end - - def move_attributes_data_to_project(start_id, end_id) - Rails.logger.info("#{self.class.name} - Moving import attributes data to projects table: #{start_id} - #{end_id}") - - if Gitlab::Database.mysql? - ActiveRecord::Base.connection.execute <<~SQL - UPDATE projects, project_mirror_data - SET - projects.import_status = project_mirror_data.status, - projects.import_jid = project_mirror_data.jid, - projects.import_error = project_mirror_data.last_error - WHERE project_mirror_data.project_id = projects.id - AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} - SQL - else - ActiveRecord::Base.connection.execute <<~SQL - UPDATE projects - SET - import_status = project_mirror_data.status, - import_jid = project_mirror_data.jid, - import_error = project_mirror_data.last_error - FROM project_mirror_data - WHERE project_mirror_data.project_id = projects.id - AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id} - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb deleted file mode 100644 index 609cf19187c..00000000000 --- a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class ScheduleDiffFilesDeletion - class MergeRequestDiff < ActiveRecord::Base - self.table_name = 'merge_request_diffs' - - belongs_to :merge_request - - include EachBatch - end - - DIFF_BATCH_SIZE = 5_000 - INTERVAL = 5.minutes - MIGRATION = 'DeleteDiffFiles' - - def perform - diffs = MergeRequestDiff - .from("(#{diffs_collection.to_sql}) merge_request_diffs") - .where('merge_request_diffs.id != merge_request_diffs.latest_merge_request_diff_id') - .select(:id) - - diffs.each_batch(of: DIFF_BATCH_SIZE) do |relation, index| - ids = relation.pluck(:id) - - BackgroundMigrationWorker.perform_in(index * INTERVAL, MIGRATION, [ids]) - end - end - - private - - def diffs_collection - MergeRequestDiff - .joins(:merge_request) - .where("merge_requests.state = 'merged'") - .where('merge_requests.latest_merge_request_diff_id IS NOT NULL') - .where("merge_request_diffs.state NOT IN ('without_files', 'empty')") - .select('merge_requests.latest_merge_request_diff_id, merge_request_diffs.id') - end - end - end -end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index fba0de20ced..6109b45ffd2 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -131,9 +131,9 @@ module Gitlab def on_109(_) set_bg_color(9, 'l') end - attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask, :sections, :lineno_in_section - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask, :sections, :lineno_in_section].freeze def convert(stream, new_state) reset_state @@ -153,10 +153,9 @@ module Gitlab start_offset = @offset - open_new_tag - stream.each_line do |line| s = StringScanner.new(line) + until s.eos? if s.scan(Gitlab::Regex.build_trace_section_regex) @@ -166,11 +165,11 @@ module Gitlab elsif s.scan(/\e(([@-_])(.*?)?)?$/) break elsif s.scan(/</) - @out << '<' + write_in_tag '<' elsif s.scan(/\r?\n/) - @out << '<br>' + handle_new_line else - @out << s.scan(/./m) + write_in_tag s.scan(/./m) end @offset += s.matched_size @@ -190,13 +189,59 @@ module Gitlab ) end + def section_to_class_name(section) + section.to_s.downcase.gsub(/[^a-z0-9]/, '-') + end + + def handle_new_line + css_classes = [] + + if @sections.any? + css_classes = %w[section line] + sections.map { |section| "s_#{section}" } + end + + write_in_tag %{<br/>} + write_raw %{<span class="#{css_classes.join(' ')}"></span>} if css_classes.any? + @lineno_in_section += 1 + open_new_tag + end + def handle_section(scanner) action = scanner[1] timestamp = scanner[2] section = scanner[3] - line = scanner.matched[0...-5] # strips \r\033[0K - @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} + normalized_section = section_to_class_name(section) + + if action == "start" + handle_section_start(normalized_section, timestamp) + elsif action == "end" + handle_section_end(normalized_section, timestamp) + end + end + + def handle_section_start(section, timestamp) + return if @sections.include?(section) + + @sections << section + write_raw %{<div class="js-section-start fa fa-caret-down append-right-8 cursor-pointer" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>} + @lineno_in_section = 0 + end + + def handle_section_end(section, timestamp) + return unless @sections.include?(section) + + # close all sections up to section + until @sections.empty? + write_raw %{<div class="section-end" data-section="#{data_section_names}"></div>} + + last_section = @sections.pop + break if section == last_section + end + end + + def data_section_names + @sections.join(" ") end def handle_sequence(scanner) @@ -217,8 +262,6 @@ module Gitlab end evaluate_command_stack(commands) - - open_new_tag end def evaluate_command_stack(stack) @@ -231,6 +274,20 @@ module Gitlab evaluate_command_stack(stack) end + def write_in_tag(data) + ensure_open_new_tag + @out << data + end + + def write_raw(data) + close_open_tags + @out << data + end + + def ensure_open_new_tag + open_new_tag if @n_open_tags == 0 + end + def open_new_tag css_classes = [] @@ -251,7 +308,11 @@ module Gitlab css_classes << "term-#{css_class}" if @style_mask & flag != 0 end - return if css_classes.empty? + if @sections.any? + css_classes << "section" + css_classes << "js-section-header" if @lineno_in_section == 0 + css_classes += sections.map { |section| "js-s-#{section}" } + end @out << %{<span class="#{css_classes.join(' ')}">} @n_open_tags += 1 @@ -268,6 +329,8 @@ module Gitlab @offset = 0 @n_open_tags = 0 @out = +'' + @sections = [] + @lineno_in_section = 0 reset end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index ecfab627226..942e4e55323 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -13,16 +13,6 @@ module Gitlab regexp = @right.evaluate(variables) regexp.scan(text.to_s).any? - - if ci_variables_complex_expressions? - # return offset of first match, or nil if no matches - if match = regexp.scan(text.to_s).first - text.to_s.index(match) - end - else - # return true or false - regexp.scan(text.to_s).any? - end end def self.build(_value, behind, ahead) @@ -32,12 +22,6 @@ module Gitlab def self.precedence 10 # See: https://ruby-doc.org/core-2.5.0/doc/syntax/precedence_rdoc.html end - - private - - def ci_variables_complex_expressions? - Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) - end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index e4cf360a1c1..eb32dffc454 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -34,7 +34,7 @@ module Gitlab end def self.eager_matching_with_escape_characters? - Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) + Feature.enabled?(:ci_variables_complex_expressions) end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 22c210ae26b..23aa1ca4a36 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -73,7 +73,7 @@ module Gitlab end def available_lexemes - Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) ? NEW_LEXEMES : LEXEMES + Feature.enabled?(:ci_variables_complex_expressions) ? NEW_LEXEMES : LEXEMES end end end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 589bf32a4d7..0679ed329e5 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -13,7 +13,7 @@ module Gitlab end def tree - if Feature.enabled?(:ci_variables_complex_expressions, default_enabled: true) + if Feature.enabled?(:ci_variables_complex_expressions) rpn_parse_tree else reverse_descent_parse_tree diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 1d55c64ec56..dcf8254ef94 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -505,7 +505,7 @@ rollout 100%: } function ensure_namespace() { - kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" } function check_kube_domain() { diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index abf16e5b2e7..8713b833011 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -31,19 +31,29 @@ sast: - | docker run \ $(propagate_env_vars \ + SAST_BANDIT_EXCLUDED_PATHS \ SAST_ANALYZER_IMAGES \ SAST_ANALYZER_IMAGE_PREFIX \ SAST_ANALYZER_IMAGE_TAG \ SAST_DEFAULT_ANALYZERS \ - SAST_EXCLUDED_PATHS \ - SAST_BANDIT_EXCLUDED_PATHS \ + SAST_PULL_ANALYZER_IMAGES \ SAST_BRAKEMAN_LEVEL \ - SAST_GOSEC_LEVEL \ SAST_FLAWFINDER_LEVEL \ SAST_GITLEAKS_ENTROPY_LEVEL \ + SAST_GOSEC_LEVEL \ + SAST_EXCLUDED_PATHS \ SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ SAST_RUN_ANALYZER_TIMEOUT \ + ANT_HOME \ + ANT_PATH \ + GRADLE_PATH \ + JAVA_OPTS \ + JAVA_PATH \ + MAVEN_CLI_OPTS \ + MAVEN_PATH \ + MAVEN_REPO_PATH \ + SBT_PATH \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/cluster/rack_timeout_observer.rb b/lib/gitlab/cluster/rack_timeout_observer.rb new file mode 100644 index 00000000000..5182b2be148 --- /dev/null +++ b/lib/gitlab/cluster/rack_timeout_observer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + class RackTimeoutObserver + TRANSITION_STATES = %i(ready active).freeze + + def initialize + @counter = Gitlab::Metrics.counter(:rack_requests_total, 'Number of requests in a given rack state') + end + + # returns the Proc to be used as the observer callback block + def callback + method(:log_timeout_exception) + end + + private + + def log_timeout_exception(env) + info = env[::Rack::Timeout::ENV_INFO_KEY] + return unless info + return if TRANSITION_STATES.include?(info.state) + + @counter.increment(labels(info, env)) + end + + def labels(info, env) + params = controller_params(env) || grape_params(env) || {} + + { + controller: params['controller'], + action: params['action'], + route: params['route'], + state: info.state + } + end + + def controller_params(env) + env['action_dispatch.request.parameters'] + end + + def grape_params(env) + endpoint = env[Grape::Env::API_ENDPOINT] + route = endpoint&.route&.pattern&.origin + return unless route + + { 'route' => route } + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/builds_event_helper.rb b/lib/gitlab/cycle_analytics/builds_event_helper.rb new file mode 100644 index 00000000000..0d6f32fdc6f --- /dev/null +++ b/lib/gitlab/cycle_analytics/builds_event_helper.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module BuildsEventHelper + def initialize(*args) + @projections = [build_table[:id]] + @order = build_table[:created_at] + + super(*args) + end + + def fetch + Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) + + super + end + + def events_query + base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + + super + end + + private + + def allowed_ids + nil + end + + def serialize(event) + AnalyticsBuildSerializer.new.represent(event['build']) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 591db3c35e6..6c348f1862d 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class CodeEventFetcher < BaseEventFetcher + include CodeHelper + def initialize(*args) @projections = [mr_table[:title], mr_table[:iid], diff --git a/lib/gitlab/cycle_analytics/code_helper.rb b/lib/gitlab/cycle_analytics/code_helper.rb new file mode 100644 index 00000000000..8f28bdd2502 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_helper.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module CodeHelper + def stage_query(project_ids) + super(project_ids).where(mr_table[:created_at].gteq(issue_metrics_table[:first_mentioned_in_commit_at])) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index 2e5f9ef5a40..89a6430221c 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class CodeStage < BaseStage + include CodeHelper + def start_time_attrs @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 30c6ead8968..8a870f2e2a3 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class IssueEventFetcher < BaseEventFetcher + include IssueHelper + def initialize(*args) @projections = [issue_table[:title], issue_table[:iid], diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb new file mode 100644 index 00000000000..c9266341378 --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_helper.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module IssueHelper + def stage_query(project_ids) + query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .project(issue_table[:project_id].as("project_id")) + .where(issue_table[:project_id].in(project_ids)) + .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + + query + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index 4eae2da512c..738cb3eba03 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class IssueStage < BaseStage + include IssueHelper + def start_time_attrs @start_time_attrs ||= issue_table[:created_at] end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index aeca9d00156..d924f956dcd 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -3,60 +3,26 @@ module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher + include PlanHelper + def initialize(*args) - @projections = [mr_diff_table[:id], - issue_metrics_table[:first_mentioned_in_commit_at]] + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id]] super(*args) end - def events_query - base_query - .join(mr_diff_table) - .on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) - - super - end - private - def allowed_ids - nil - end - - def merge_request_diff_commits - @merge_request_diff_commits ||= - MergeRequestDiffCommit - .where(merge_request_diff_id: event_result.map { |event| event['id'] }) - .group_by(&:merge_request_diff_id) - end - def serialize(event) - commit = first_time_reference_commit(event) - - return unless commit - - serialize_commit(event, commit, query) - end - - def first_time_reference_commit(event) - return unless event && merge_request_diff_commits - - commits = merge_request_diff_commits[event['id'].to_i] - - return if commits.blank? - - commits.find do |commit| - next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] - - commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i - end + AnalyticsIssueSerializer.new(project: @project).represent(event) end - def serialize_commit(event, commit, query) - commit = Commit.from_hash(commit.to_hash, @project) - - AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) + def allowed_ids_finder_class + IssuesFinder end end end diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb new file mode 100644 index 00000000000..30fc2ce6d40 --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_helper.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module PlanHelper + def stage_query(project_ids) + query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .project(issue_table[:project_id].as("project_id")) + .where(issue_table[:project_id].in(project_ids)) + .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) + + query + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 513e4575be0..0b27d114f52 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class PlanStage < BaseStage + include PlanHelper + def start_time_attrs @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], issue_metrics_table[:first_added_to_board_at]] @@ -21,7 +23,7 @@ module Gitlab end def legend - _("Related Commits") + _("Related Issues") end def description diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb index 6681cb42c90..6bcbe0412a9 100644 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -2,7 +2,28 @@ module Gitlab module CycleAnalytics - class ProductionEventFetcher < IssueEventFetcher + class ProductionEventFetcher < BaseEventFetcher + include ProductionHelper + + def initialize(*args) + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id]] + + super(*args) + end + + private + + def serialize(event) + AnalyticsIssueSerializer.new(project: @project).represent(event) + end + + def allowed_ids_finder_class + IssuesFinder + end end end end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index de100295281..b6354b5ffad 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class ReviewEventFetcher < BaseEventFetcher + include ReviewHelper + def initialize(*args) @projections = [mr_table[:title], mr_table[:iid], diff --git a/lib/gitlab/cycle_analytics/review_helper.rb b/lib/gitlab/cycle_analytics/review_helper.rb new file mode 100644 index 00000000000..c53249652b5 --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_helper.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module ReviewHelper + def stage_query(project_ids) + super(project_ids).where(mr_metrics_table[:merged_at].not_eq(nil)) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index 294b656bc55..e9df8cd5a05 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class ReviewStage < BaseStage + include ReviewHelper + def start_time_attrs @start_time_attrs ||= mr_table[:created_at] end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index 70ce82383b3..1454a1a33eb 100644 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -3,34 +3,8 @@ module Gitlab module CycleAnalytics class StagingEventFetcher < BaseEventFetcher - def initialize(*args) - @projections = [build_table[:id]] - @order = build_table[:created_at] - - super(*args) - end - - def fetch - Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) - - super - end - - def events_query - base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - - super - end - - private - - def allowed_ids - nil - end - - def serialize(event) - AnalyticsBuildSerializer.new.represent(event['build']) - end + include ProductionHelper + include BuildsEventHelper end end end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index dbc2414ff66..e03627c6cd1 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class StagingStage < BaseStage include ProductionHelper + def start_time_attrs @start_time_attrs ||= mr_metrics_table[:merged_at] end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb index 4d5ea5b7c34..2fa44b1b364 100644 --- a/lib/gitlab/cycle_analytics/test_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -2,7 +2,9 @@ module Gitlab module CycleAnalytics - class TestEventFetcher < StagingEventFetcher + class TestEventFetcher < BaseEventFetcher + include TestHelper + include BuildsEventHelper end end end diff --git a/lib/gitlab/cycle_analytics/test_helper.rb b/lib/gitlab/cycle_analytics/test_helper.rb new file mode 100644 index 00000000000..32fca7fa898 --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_helper.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module TestHelper + def stage_query(project_ids) + if branch + super(project_ids).where(build_table[:ref].eq(branch)) + else + super(project_ids) + end + end + + private + + def branch + @branch ||= @options[:branch] # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index c31b664148b..4787a906c07 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -3,6 +3,8 @@ module Gitlab module CycleAnalytics class TestStage < BaseStage + include TestHelper + def start_time_attrs @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] end @@ -26,14 +28,6 @@ module Gitlab def description _("Total test time for all commits/merges") end - - def stage_query(project_ids) - if @options[:branch] - super(project_ids).where(build_table[:ref].eq(@options[:branch])) - else - super(project_ids) - end - end end end end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 7a0fb419f8e..1ecf4a12db7 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -124,6 +124,10 @@ module Gitlab %r{\.(md|txt)\z} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\.js\z} => :frontend }.freeze + + def new_teammates(usernames) + usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } + end end end end diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index 062eda38ee4..25de0a87c9d 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -45,21 +45,19 @@ module Gitlab # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the # selection will change on next spin def spin_for_person(people, random:) - person = nil - people = people.dup - - people.size.times do - person = people.sample(random: random) - - break person unless out_of_office?(person) + people.shuffle(random: random) + .find(&method(:valid_person?)) + end - people -= [person] - end + private - person + def valid_person?(person) + !mr_author?(person) && !out_of_office?(person) end - private + def mr_author?(person) + person.username == gitlab.mr_author + end def out_of_office?(person) username = CGI.escape(person.username) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index cc61bb7fa02..1b5cd0fbb07 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -149,7 +149,7 @@ module Gitlab # column - The name of the column to create the foreign key on. # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade) + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -167,14 +167,18 @@ module Gitlab return end - return add_foreign_key(source, target, - column: column, - on_delete: on_delete) + key_options = { column: column, on_delete: on_delete } + + # The MySQL adapter tries to create a foreign key without a name when + # `:name` is nil, instead of generating a name for us. + key_options[:name] = name if name + + return add_foreign_key(source, target, key_options) else on_delete = 'SET NULL' if on_delete == :nullify end - key_name = concurrent_foreign_key_name(source, column) + key_name = name || concurrent_foreign_key_name(source, column) unless foreign_key_exists?(source, target, column: column) Rails.logger.warn "Foreign key not created because it exists already " \ diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index e8f98f52111..d349c378e53 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -134,6 +134,14 @@ module Gitlab @line_code ||= diff_file(repository)&.line_code_for_position(self) end + def on_image? + position_type == 'image' + end + + def on_text? + position_type == 'text' + end + private def find_diff_file(repository) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 5ff415b6126..1d317c389d2 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -52,12 +52,13 @@ module Gitlab def using_keychain Gitlab::Gpg.using_tmp_keychain do - # first we need to get the keyid from the signature to query the gpg - # key belonging to the keyid. + # first we need to get the fingerprint from the signature to query the gpg + # key belonging to the fingerprint. # This way we can add the key to the temporary keychain and extract # the proper signature. - # NOTE: the invoked method is #fingerprint but it's only returning - # 16 characters (the format used by keyid) instead of 40. + # NOTE: the invoked method is #fingerprint but versions of GnuPG + # prior to 2.2.13 return 16 characters (the format used by keyid) + # instead of 40. fingerprint = verified_signature&.fingerprint break unless fingerprint @@ -128,11 +129,13 @@ module Gitlab gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end - # rubocop: disable CodeReuse/ActiveRecord - def find_gpg_key(keyid) - GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) + def find_gpg_key(fingerprint) + if fingerprint.length > 16 + GpgKey.find_by_fingerprint(fingerprint) || GpgKeySubkey.find_by_fingerprint(fingerprint) + else + GpgKey.find_by_primary_keyid(fingerprint) || GpgKeySubkey.find_by_keyid(fingerprint) + end end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb new file mode 100644 index 00000000000..f6cd4eb5e0c --- /dev/null +++ b/lib/gitlab/import_export/config.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class Config + # Returns a Hash of the YAML file, including EE specific data if EE is + # used. + def to_h + hash = parse_yaml + ee_hash = hash['ee'] + + if merge? && ee_hash + ee_hash.each do |key, value| + if key == 'project_tree' + merge_project_tree(value, hash[key]) + else + merge_attributes_list(value, hash[key]) + end + end + end + + # We don't want to expose this section after this point, as it is no + # longer needed. + hash.delete('ee') + + hash + end + + # Merges a project relationships tree into the target tree. + # + # @param [Array<Hash|Symbol>] source_values + # @param [Array<Hash|Symbol>] target_values + def merge_project_tree(source_values, target_values) + source_values.each do |value| + if value.is_a?(Hash) + # Examples: + # + # { 'project_tree' => [{ 'labels' => [...] }] } + # { 'notes' => [:author, { 'events' => [:push_event_payload] }] } + value.each do |key, val| + target = target_values + .find { |h| h.is_a?(Hash) && h[key] } + + if target + merge_project_tree(val, target[key]) + else + target_values << { key => val.dup } + end + end + else + # Example: :priorities, :author, etc + target_values << value + end + end + end + + # Merges a Hash containing a flat list of attributes, such as the entries + # in a `excluded_attributes` section. + # + # @param [Hash] source_values + # @param [Hash] target_values + def merge_attributes_list(source_values, target_values) + source_values.each do |key, values| + target_values[key] ||= [] + target_values[key].concat(values) + end + end + + def merge? + Gitlab.ee? + end + + def parse_yaml + YAML.load_file(Gitlab::ImportExport.config_file) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 71c44af9254..a0fb051e806 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,7 +1,11 @@ # Model relationships to be included in the project import/export +# +# This list _must_ only contain relationships that are available to both CE and +# EE. EE specific relationships must be defined in the `ee` section further +# down below. project_tree: - labels: - :priorities + - :priorities - milestones: - events: - :push_event_payload @@ -15,18 +19,18 @@ project_tree: - :push_event_payload - label_links: - label: - :priorities + - :priorities - milestone: - events: - :push_event_payload - resource_label_events: - label: - :priorities + - :priorities - :issue_assignees - snippets: - :award_emoji - notes: - :author + - :author - releases: - :links - project_members: @@ -46,13 +50,13 @@ project_tree: - :timelogs - label_links: - label: - :priorities + - :priorities - milestone: - events: - :push_event_payload - resource_label_events: - label: - :priorities + - :priorities - ci_pipelines: - notes: - :author @@ -121,12 +125,22 @@ excluded_attributes: - :bfg_object_map - :detected_repository_languages - :tag_list + - :mirror_user_id + - :mirror_trigger_builds + - :only_mirror_protected_branches + - :pull_mirror_available_overridden + - :mirror_overwrites_diverged_branches + - :packages_enabled + - :mirror_last_update_at + - :mirror_last_successful_update_at namespaces: - :runners_token - :runners_token_encrypted project_import_state: - :last_error - :jid + - :last_update_at + - :last_successful_update_at prometheus_metrics: - :common - :identifier @@ -201,3 +215,12 @@ methods: - :action project_badges: - :type + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + project_tree: + - protected_branches: + - :unprotect_access_levels + - protected_environments: + - :deploy_access_levels diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index bc0d18e03fa..8bdf6ca491d 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -7,7 +7,7 @@ module Gitlab def initialize(shared:) @shared = shared - config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + config_hash = ImportExport::Config.new.to_h.deep_symbolize_keys @tree = config_hash[:project_tree] @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], excluded_attributes: config_hash[:excluded_attributes], diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 70b18221a66..751726d4810 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -55,7 +55,12 @@ module Gitlab import_pull_requests import_issues import_comments(:issues) - import_comments(:pull_requests) + + # Gitea doesn't have an API endpoint for pull requests comments + unless project.gitea_import? + import_comments(:pull_requests) + end + import_wiki # Gitea doesn't have a Release API yet diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb index 4664aee71f6..090014aa597 100644 --- a/lib/gitlab/metrics/dashboard/base_service.rb +++ b/lib/gitlab/metrics/dashboard/base_service.rb @@ -7,18 +7,19 @@ module Gitlab module Dashboard class BaseService < ::BaseService PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError + NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError def get_dashboard - return error("#{dashboard_path} could not be found.", :not_found) unless path_available? - success(dashboard: process_dashboard) + rescue NOT_FOUND_ERROR + error("#{dashboard_path} could not be found.", :not_found) rescue PROCESSING_ERROR => e error(e.message, :unprocessable_entity) end # Summary of all known dashboards for the service. # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] - def all_dashboard_paths(_project) + def self.all_dashboard_paths(_project) raise NotImplementedError end @@ -38,7 +39,7 @@ module Gitlab # Returns an un-processed dashboard from the cache. def raw_dashboard - Rails.cache.fetch(cache_key) { get_raw_dashboard } + Gitlab::Metrics::Dashboard::Cache.fetch(cache_key) { get_raw_dashboard } end # @return [Hash] an unmodified dashboard @@ -56,17 +57,6 @@ module Gitlab def insert_project_metrics? false end - - # Checks if dashboard path exists or should be rejected - # as a result of file-changes to the project repository. - # @return [Boolean] - def path_available? - available_paths = Gitlab::Metrics::Dashboard::Finder.find_all_paths(project) - - available_paths.any? do |path_params| - path_params[:path] == dashboard_path - end - end end end end diff --git a/lib/gitlab/metrics/dashboard/cache.rb b/lib/gitlab/metrics/dashboard/cache.rb new file mode 100644 index 00000000000..a9ccf0fea9b --- /dev/null +++ b/lib/gitlab/metrics/dashboard/cache.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'set' + +module Gitlab + module Metrics + module Dashboard + class Cache + CACHE_KEYS = 'all_cached_metric_dashboards' + + class << self + # Stores a dashboard in the cache, documenting the key + # so the cached can be cleared in bulk at another time. + def fetch(key) + register_key(key) + + Rails.cache.fetch(key) { yield } + end + + # Resets all dashboard caches, such that all + # dashboard content will be loaded from source on + # subsequent dashboard calls. + def delete_all! + all_keys.each { |key| Rails.cache.delete(key) } + + Rails.cache.delete(CACHE_KEYS) + end + + private + + def register_key(key) + new_keys = all_keys.add(key).to_a.join('|') + + Rails.cache.write(CACHE_KEYS, new_keys) + end + + def all_keys + Set.new(Rails.cache.read(CACHE_KEYS)&.split('|')) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 4a41590f000..49ea5c7d4f2 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -27,6 +27,8 @@ module Gitlab # Summary of all known dashboards. Used to populate repo cache. # Prefer #find_all_paths. def find_all_paths_from_source(project) + Gitlab::Metrics::Dashboard::Cache.delete_all! + system_service.all_dashboard_paths(project) .+ project_service.all_dashboard_paths(project) end diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb index fdffd067c93..5a1c4ecf886 100644 --- a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb +++ b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb @@ -14,9 +14,11 @@ module Gitlab file_finder(project) .list_files_for(DASHBOARD_ROOT) .map do |filepath| - Rails.cache.delete(cache_key(project.id, filepath)) - - { path: filepath, default: false } + { + path: filepath, + display_name: name_for_path(filepath), + default: false + } end end @@ -24,8 +26,9 @@ module Gitlab Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') end - def cache_key(id, dashboard_path) - "project_#{id}_metrics_dashboard_#{dashboard_path}" + # Grabs the filepath after the base directory. + def name_for_path(filepath) + filepath.delete_prefix("#{DASHBOARD_ROOT}/") end end @@ -39,7 +42,7 @@ module Gitlab end def cache_key - self.class.cache_key(project.id, dashboard_path) + "project_#{project.id}_metrics_dashboard_#{dashboard_path}" end end end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb index 67509ed4230..82421572f4a 100644 --- a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb +++ b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb @@ -7,11 +7,13 @@ module Gitlab module Dashboard class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' + SYSTEM_DASHBOARD_NAME = 'Default' class << self def all_dashboard_paths(_project) [{ path: SYSTEM_DASHBOARD_PATH, + display_name: SYSTEM_DASHBOARD_NAME, default: true }] end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 83204fa5d18..2a2083ebae0 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -63,12 +63,6 @@ module Gitlab { remote_sign_out_handler: authentiq_signout_handler } when 'shibboleth' { fail_with_empty_uid: true } - when 'openid_connect' - # If a name argument is omitted, OmniAuth will expect that the - # matching route is /auth/users/openidconnect instead of - # /auth/users/openid_connect because of - # https://gitlab.com/gitlab-org/gitlab-ce/issues/62208#note_178780341. - { name: 'openid_connect' } else {} end diff --git a/lib/gitlab/rack_timeout_observer.rb b/lib/gitlab/rack_timeout_observer.rb deleted file mode 100644 index 80d3f7dea60..00000000000 --- a/lib/gitlab/rack_timeout_observer.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class RackTimeoutObserver - def initialize - @counter = Gitlab::Metrics.counter(:rack_state_total, 'Number of requests in a given rack state') - end - - # returns the Proc to be used as the observer callback block - def callback - method(:log_timeout_exception) - end - - private - - def log_timeout_exception(env) - info = env[::Rack::Timeout::ENV_INFO_KEY] - return unless info - - @counter.increment(labels(info, env)) - end - - def labels(info, env) - params = controller_params(env) || grape_params(env) || {} - - { - controller: params['controller'], - action: params['action'], - route: params['route'], - state: info.state - } - end - - def controller_params(env) - env['action_dispatch.request.parameters'] - end - - def grape_params(env) - endpoint = env[Grape::Env::API_ENDPOINT] - route = endpoint&.route&.pattern&.origin - return unless route - - { 'route' => route } - end - end -end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 3daa03d01d6..7c1e6b1baff 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -85,10 +85,6 @@ module Gitlab UsersFinder.new(current_user, search: query).execute end - def display_options(_scope) - {} - end - private def projects diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 8f9d5cf1e63..e2787744f09 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -138,5 +138,18 @@ module Gitlab def visibility=(level) self[visibility_level_field] = Gitlab::VisibilityLevel.level_value(level) end + + def visibility_attribute_present?(attributes) + visibility_level_attributes.each do |attr| + return true if attributes[attr].present? + end + + false + end + + def visibility_level_attributes + [visibility_level_field, visibility_level_field.to_s, + :visibility, 'visibility'] + end end end diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 900dbf7be24..5365bd3920f 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -7,7 +7,7 @@ namespace :gitlab do desc "GitLab | Display exported DB structure" task data: :environment do - puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) + puts Gitlab::ImportExport::Config.new.to_h['project_tree'].to_yaml(SortKeys: true) end desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' |