diff options
Diffstat (limited to 'lib')
77 files changed, 868 insertions, 1242 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 20f8c637274..7016a66593d 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -52,7 +52,10 @@ module API rack_response({ 'message' => '404 Not found' }.to_json, 404) end - rescue_from ::Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError do + rescue_from( + ::ActiveRecord::StaleObjectError, + ::Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError + ) do rack_response({ 'message' => '409 Conflict: Resource lock' }.to_json, 409) end @@ -163,6 +166,7 @@ module API mount ::API::Templates mount ::API::Todos mount ::API::Triggers + mount ::API::UserCounts mount ::API::Users mount ::API::Variables mount ::API::Version diff --git a/lib/api/entities.rb b/lib/api/entities.rb index d783591c238..b9aa387ba61 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -498,9 +498,9 @@ module API expose :state, :created_at, :updated_at # Avoids an N+1 query when metadata is included - def issuable_metadata(subject, options, method) + def issuable_metadata(subject, options, method, args = nil) cached_subject = options.dig(:issuable_metadata, subject.id) - (cached_subject || subject).public_send(method) # rubocop: disable GitlabSecurity/PublicSend + (cached_subject || subject).public_send(method, *args) # rubocop: disable GitlabSecurity/PublicSend end end @@ -564,7 +564,7 @@ module API end expose(:user_notes_count) { |issue, options| issuable_metadata(issue, options, :user_notes_count) } - expose(:merge_requests_count) { |issue, options| issuable_metadata(issue, options, :merge_requests_count) } + expose(:merge_requests_count) { |issue, options| issuable_metadata(issue, options, :merge_requests_count, options[:current_user]) } expose(:upvotes) { |issue, options| issuable_metadata(issue, options, :upvotes) } expose(:downvotes) { |issue, options| issuable_metadata(issue, options, :downvotes) } expose :due_date @@ -757,7 +757,9 @@ module API merge_request.metrics&.pipeline end - expose :head_pipeline, using: 'API::Entities::Pipeline' + expose :head_pipeline, using: 'API::Entities::Pipeline', if: -> (_, options) do + Ability.allowed?(options[:current_user], :read_pipeline, options[:project]) + end expose :diff_refs, using: Entities::DiffRefs @@ -1186,8 +1188,10 @@ module API MarkupHelper.markdown_field(entity, :description) end expose :created_at + expose :released_at expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } expose :commit, using: Entities::Commit, if: lambda { |_, _| can_download_code? } + expose :upcoming_release?, as: :upcoming_release expose :assets do expose :assets_count, as: :count do |release, _| diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb index 94010ab1bc2..bd60470fbd6 100644 --- a/lib/api/helpers/graphql_helpers.rb +++ b/lib/api/helpers/graphql_helpers.rb @@ -7,8 +7,6 @@ module API # should be in app/graphql/ or lib/gitlab/graphql/ module GraphqlHelpers def conditionally_graphql!(fallback:, query:, context: {}, transform: nil) - return fallback.call unless Feature.enabled?(:graphql) - result = GitlabSchema.execute(query, context: context) if transform diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 039ebf92187..d687acf3423 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -96,7 +96,7 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue') + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) } present issues, options @@ -122,7 +122,7 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue') + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) } present issues, options @@ -161,7 +161,7 @@ module API with_labels_details: declared_params[:with_labels_details], current_user: current_user, project: user_project, - issuable_metadata: issuable_meta_data(issues, 'Issue') + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) } present issues, options diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index bf87e9ec2ff..64ee82cd775 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -72,7 +72,7 @@ module API if params[:view] == 'simple' options[:with] = Entities::MergeRequestSimple else - options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest', current_user) end options @@ -429,9 +429,10 @@ module API authorize_push_to_merge_request!(merge_request) - RebaseWorker.perform_async(merge_request.id, current_user.id) + merge_request.rebase_async(current_user.id) status :accepted + present rebase_in_progress: merge_request.rebase_in_progress? end desc 'List issues that will be closed on merge' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 1e14c77b5be..a7d62014509 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -474,7 +474,7 @@ module API authorize_admin_project begin - ::Projects::HousekeepingService.new(user_project).execute + ::Projects::HousekeepingService.new(user_project, :gc).execute rescue ::Projects::HousekeepingService::LeaseTaken => error conflict!(error.message) end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 6b17f4317db..fdd8406388e 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -54,6 +54,7 @@ module API requires :url, type: String end end + optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready. Defaults to the current time.' end post ':id/releases' do authorize_create_release! @@ -77,6 +78,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag', as: :tag optional :name, type: String, desc: 'The name of the release' optional :description, type: String, desc: 'Release notes with markdown support' + optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready. Defaults to the current time.' end put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do authorize_update_release! diff --git a/lib/api/runners.rb b/lib/api/runners.rb index f3fea463e7f..c2d371b6867 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -115,6 +115,8 @@ module API params do requires :id, type: Integer, desc: 'The ID of the runner' optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES + optional :order_by, type: String, desc: 'Order by `id` or not', values: RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS + optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end get ':id/jobs' do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 3c5c1a9fd5f..4275d911708 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -55,6 +55,8 @@ module API optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :grafana_enabled, type: Boolean, desc: 'Enable Grafana' + optional :grafana_url, type: String, desc: 'Grafana URL' optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help' optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page' diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 871eaabc887..7260ecfb5ee 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -65,7 +65,7 @@ module API next unless collection targets = collection.map(&:target) - options[type] = { issuable_metadata: issuable_meta_data(targets, type) } + options[type] = { issuable_metadata: issuable_meta_data(targets, type, current_user) } end end end diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb new file mode 100644 index 00000000000..8df4b381bbf --- /dev/null +++ b/lib/api/user_counts.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module API + class UserCounts < Grape::API + resource :user_counts do + desc 'Return the user specific counts' do + detail 'Open MR Count' + end + get do + unauthorized! unless current_user + + { + merge_requests: current_user.assigned_open_merge_requests_count + } + end + end + end +end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 80c84c0f622..86f18679496 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -56,10 +56,10 @@ module Banzai def process_link_to_upload_attr(html_attr) path_parts = [Addressable::URI.unescape(html_attr.value)] - if group - path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') - elsif project + if project path_parts.unshift(relative_url_root, project.full_path) + elsif group + path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') else path_parts.unshift(relative_url_root) end @@ -102,7 +102,7 @@ module Banzai end def relative_file_path(uri) - path = Addressable::URI.unescape(uri.path) + path = Addressable::URI.unescape(uri.path).delete("\0") request_path = Addressable::URI.unescape(context[:requested_path]) nested_path = build_relative_path(path, request_path) file_exists?(nested_path) ? nested_path : path diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index fe56f9a1e33..9b66759a5fb 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,7 +14,7 @@ module Banzai LANG_PARAMS_ATTR = 'data-lang-params'.freeze def call - doc.search('pre > code').each do |node| + doc.search('pre:not([data-math-style]) > code').each do |node| highlight_node(node) end diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index cc4af280872..6be489c6572 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -6,6 +6,7 @@ module Banzai def self.filters FilterArray[ Filter::SanitizationFilter, + Filter::SyntaxHighlightFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, Filter::AsciiDocPostProcessingFilter diff --git a/lib/feature.rb b/lib/feature.rb index cc9c9d44005..e28333aa58e 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -34,7 +34,9 @@ class Feature begin # We saw on GitLab.com, this database request was called 2300 # times/s. Let's cache it for a minute to avoid that load. - Rails.cache.fetch('flipper:persisted_names', expires_in: 1.minute) { FlipperFeature.feature_names } + Gitlab::ThreadMemoryCache.cache_backend.fetch('flipper:persisted_names', expires_in: 1.minute) do + FlipperFeature.feature_names + end end end @@ -101,10 +103,27 @@ class Feature feature_class: FlipperFeature, gate_class: FlipperGate) + # Redis L2 cache + redis_cache_adapter = + Flipper::Adapters::ActiveSupportCacheStore.new( + active_record_adapter, + l2_cache_backend, + expires_in: 1.hour) + + # Thread-local L1 cache: use a short timeout since we don't have a + # way to expire this cache all at once Flipper::Adapters::ActiveSupportCacheStore.new( - active_record_adapter, - Rails.cache, - expires_in: 1.hour) + redis_cache_adapter, + l1_cache_backend, + expires_in: 1.minute) + end + + def l1_cache_backend + Gitlab::ThreadMemoryCache.cache_backend + end + + def l2_cache_backend + Rails.cache end end diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index d7a8f8a0b9e..67c0b902c0c 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -8,7 +8,12 @@ class Feature # CATFILE_CACHE sets an incorrect example CATFILE_CACHE = 'catfile-cache'.freeze - SERVER_FEATURE_FLAGS = [CATFILE_CACHE].freeze + SERVER_FEATURE_FLAGS = + [ + CATFILE_CACHE, + 'get_commit_signatures'.freeze + ].freeze + DEFAULT_ON_FLAGS = Set.new([CATFILE_CACHE]).freeze class << self diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 7f8300a0c2f..00c87cce7b6 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -4,6 +4,7 @@ require 'asciidoctor' require 'asciidoctor-plantuml' require 'asciidoctor/extensions' require 'gitlab/asciidoc/html5_converter' +require 'gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -16,7 +17,7 @@ module Gitlab 'idseparator' => '-', 'env' => 'gitlab', 'env-gitlab' => '', - 'source-highlighter' => 'html-pipeline', + 'source-highlighter' => 'gitlab-html-pipeline', 'icons' => 'font', 'outfilesuffix' => '.adoc', 'max-include-depth' => MAX_INCLUDE_DEPTH diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb new file mode 100644 index 00000000000..5fc3323f0fd --- /dev/null +++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Asciidoc + module SyntaxHighlighter + class HtmlPipelineAdapter < Asciidoctor::SyntaxHighlighter::Base + register_for 'gitlab-html-pipeline' + + def format(node, lang, opts) + %(<pre><code #{lang ? %[ lang="#{lang}"] : ''}>#{node.content}</code></pre>) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb deleted file mode 100644 index ccd1f9b4dba..00000000000 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class CreateForkNetworkMembershipsRange - RESCHEDULE_DELAY = 15 - - class ForkedProjectLink < ActiveRecord::Base - self.table_name = 'forked_project_links' - end - - def perform(start_id, end_id) - log("Creating memberships for forks: #{start_id} - #{end_id}") - - insert_members(start_id, end_id) - - if missing_members?(start_id, end_id) - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) - end - end - - def insert_members(start_id, end_id) - ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS - INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) - - SELECT fork_network_members.fork_network_id, - forked_project_links.forked_to_project_id, - forked_project_links.forked_from_project_id - - FROM forked_project_links - - INNER JOIN fork_network_members - ON forked_project_links.forked_from_project_id = fork_network_members.project_id - - WHERE forked_project_links.id BETWEEN #{start_id} AND #{end_id} - AND NOT EXISTS ( - SELECT true - FROM fork_network_members existing_members - WHERE existing_members.project_id = forked_project_links.forked_to_project_id - ) - INSERT_MEMBERS - rescue ActiveRecord::RecordNotUnique => e - # `fork_network_member` was created concurrently in another migration - log(e.message) - end - - def missing_members?(start_id, end_id) - count_sql = <<~MISSING_MEMBERS - SELECT COUNT(*) - - FROM forked_project_links - - WHERE NOT EXISTS ( - SELECT true - FROM fork_network_members - WHERE fork_network_members.project_id = forked_project_links.forked_to_project_id - ) - AND EXISTS ( - SELECT true - FROM projects - WHERE forked_project_links.forked_from_project_id = projects.id - ) - AND NOT EXISTS ( - SELECT true - FROM forked_project_links AS parent_links - WHERE parent_links.forked_to_project_id = forked_project_links.forked_from_project_id - AND NOT EXISTS ( - SELECT true - FROM projects - WHERE parent_links.forked_from_project_id = projects.id - ) - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - MISSING_MEMBERS - - ForkedProjectLink.count_by_sql(count_sql) > 0 - end - - def log(message) - Rails.logger.info("#{self.class.name} - #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb deleted file mode 100644 index 21b626dde56..00000000000 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class DeleteConflictingRedirectRoutesRange - def perform(start_id, end_id) - # No-op. - # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb deleted file mode 100644 index 42fcaa87e66..00000000000 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ /dev/null @@ -1,179 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - # Class that migrates events for the new push event payloads setup. All - # events are copied to a shadow table, and push events will also have a row - # created in the push_event_payloads table. - class MigrateEventsToPushEventPayloads - class Event < ActiveRecord::Base - self.table_name = 'events' - - serialize :data - - BLANK_REF = ('0' * 40).freeze - TAG_REF_PREFIX = 'refs/tags/'.freeze - MAX_INDEX = 69 - PUSHED = 5 - - def push_event? - action == PUSHED && data.present? - end - - def commit_title - commit = commits.last - - return unless commit && commit[:message] - - index = commit[:message].index("\n") - message = index ? commit[:message][0..index] : commit[:message] - - message.strip.truncate(70) - end - - def commit_from_sha - if create? - nil - else - data[:before] - end - end - - def commit_to_sha - if remove? - nil - else - data[:after] - end - end - - def data - super || {} - end - - def commits - data[:commits] || [] - end - - def commit_count - data[:total_commits_count] || 0 - end - - def ref - data[:ref] - end - - def trimmed_ref_name - if ref_type == :tag - ref[10..-1] - else - ref[11..-1] - end - end - - def create? - data[:before] == BLANK_REF - end - - def remove? - data[:after] == BLANK_REF - end - - def push_action - if create? - :created - elsif remove? - :removed - else - :pushed - end - end - - def ref_type - if ref.start_with?(TAG_REF_PREFIX) - :tag - else - :branch - end - end - end - - class EventForMigration < ActiveRecord::Base - self.table_name = 'events_for_migration' - end - - class PushEventPayload < ActiveRecord::Base - self.table_name = 'push_event_payloads' - - enum action: { - created: 0, - removed: 1, - pushed: 2 - } - - enum ref_type: { - branch: 0, - tag: 1 - } - end - - # start_id - The start ID of the range of events to process - # end_id - The end ID of the range to process. - def perform(start_id, end_id) - return unless migrate? - - find_events(start_id, end_id).each { |event| process_event(event) } - end - - def process_event(event) - ActiveRecord::Base.transaction do - replicate_event(event) - create_push_event_payload(event) if event.push_event? - end - rescue ActiveRecord::InvalidForeignKey => e - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. - Rails.logger.error("Unable to migrate event #{event.id}: #{e}") - end - - def replicate_event(event) - new_attributes = event.attributes - .with_indifferent_access.except(:title, :data) - - EventForMigration.create!(new_attributes) - end - - def create_push_event_payload(event) - commit_from = pack(event.commit_from_sha) - commit_to = pack(event.commit_to_sha) - - PushEventPayload.create!( - event_id: event.id, - commit_count: event.commit_count, - ref_type: event.ref_type, - action: event.push_action, - commit_from: commit_from, - commit_to: commit_to, - ref: event.trimmed_ref_name, - commit_title: event.commit_title - ) - end - - def find_events(start_id, end_id) - Event - .where('NOT EXISTS (SELECT true FROM events_for_migration WHERE events_for_migration.id = events.id)') - .where(id: start_id..end_id) - end - - def migrate? - Event.table_exists? && PushEventPayload.table_exists? && - EventForMigration.table_exists? - end - - def pack(value) - value ? [value].pack('H*') : nil - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb deleted file mode 100644 index ef50fe4adb1..00000000000 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateSystemUploadsToNewFolder - include Gitlab::Database::MigrationHelpers - attr_reader :old_folder, :new_folder - - class Upload < ActiveRecord::Base - self.table_name = 'uploads' - include EachBatch - end - - def perform(old_folder, new_folder) - replace_sql = replace_sql(uploads[:path], old_folder, new_folder) - affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%")) - - affected_uploads.each_batch do |batch| - batch.update_all("path = #{replace_sql}") - end - end - - def uploads - Arel::Table.new('uploads') - end - end - end -end diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb deleted file mode 100644 index 5b2b2af718a..00000000000 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MovePersonalSnippetFiles - delegate :select_all, :execute, :quote_string, to: :connection - - def perform(relative_source, relative_destination) - @source_relative_location = relative_source - @destination_relative_location = relative_destination - - move_personal_snippet_files - end - - def move_personal_snippet_files - query = "SELECT uploads.path, uploads.model_id FROM uploads "\ - "INNER JOIN snippets ON snippets.id = uploads.model_id WHERE uploader = 'PersonalFileUploader'" - select_all(query).each do |upload| - secret = upload['path'].split('/')[0] - file_name = upload['path'].split('/')[1] - - move_file(upload['model_id'], secret, file_name) - update_markdown(upload['model_id'], secret, file_name) - end - end - - def move_file(snippet_id, secret, file_name) - source_dir = File.join(base_directory, @source_relative_location, snippet_id.to_s, secret) - destination_dir = File.join(base_directory, @destination_relative_location, snippet_id.to_s, secret) - - source_file_path = File.join(source_dir, file_name) - destination_file_path = File.join(destination_dir, file_name) - - unless File.exist?(source_file_path) - say "Source file `#{source_file_path}` doesn't exist. Skipping." - return - end - - say "Moving file #{source_file_path} -> #{destination_file_path}" - - FileUtils.mkdir_p(destination_dir) - FileUtils.move(source_file_path, destination_file_path) - end - - def update_markdown(snippet_id, secret, file_name) - source_markdown_path = File.join(@source_relative_location, snippet_id.to_s, secret, file_name) - destination_markdown_path = File.join(@destination_relative_location, snippet_id.to_s, secret, file_name) - - source_markdown = "](#{source_markdown_path})" - destination_markdown = "](#{destination_markdown_path})" - quoted_source = quote_string(source_markdown) - quoted_destination = quote_string(destination_markdown) - - execute("UPDATE snippets "\ - "SET description = replace(snippets.description, '#{quoted_source}', '#{quoted_destination}'), description_html = NULL "\ - "WHERE id = #{snippet_id}") - - query = "SELECT id, note FROM notes WHERE noteable_id = #{snippet_id} "\ - "AND noteable_type = 'Snippet' AND note IS NOT NULL" - select_all(query).each do |note| - text = note['note'].gsub(source_markdown, destination_markdown) - quoted_text = quote_string(text) - - execute("UPDATE notes SET note = '#{quoted_text}', note_html = NULL WHERE id = #{note['id']}") - end - end - - def base_directory - File.join(Rails.root, 'public') - end - - def connection - ActiveRecord::Base.connection - end - - def say(message) - Rails.logger.debug(message) - end - end - end -end diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb deleted file mode 100644 index 48aa369705f..00000000000 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ /dev/null @@ -1,319 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/ClassLength -# rubocop:disable Metrics/BlockLength -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class NormalizeLdapExternUidsRange - class Identity < ActiveRecord::Base - self.table_name = 'identities' - end - - # Copied this class to make this migration resilient to future code changes. - # And if the normalize behavior is changed in the future, it must be - # accompanied by another migration. - module Gitlab - module Auth - module LDAP - class DN - FormatError = Class.new(StandardError) - MalformedError = Class.new(FormatError) - UnsupportedError = Class.new(FormatError) - - def self.normalize_value(given_value) - dummy_dn = "placeholder=#{given_value}" - normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') - end - - ## - # Initialize a DN, escaping as required. Pass in attributes in name/value - # pairs. If there is a left over argument, it will be appended to the dn - # without escaping (useful for a base string). - # - # Most uses of this class will be to escape a DN, rather than to parse it, - # so storing the dn as an escaped String and parsing parts as required - # with a state machine seems sensible. - def initialize(*args) - if args.length > 1 - initialize_array(args) - else - initialize_string(args[0]) - end - end - - ## - # Parse a DN into key value pairs using ASN from - # http://tools.ietf.org/html/rfc2253 section 3. - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def each_pair - state = :key - key = StringIO.new - value = StringIO.new - hex_buffer = "" - - @dn.each_char.with_index do |char, dn_index| - case state - when :key then - case char - when 'a'..'z', 'A'..'Z' then - state = :key_normal - key << char - when '0'..'9' then - state = :key_oid - key << char - when ' ' then state = :key - else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") - end - when :key_normal then - case char - when '=' then state = :value - when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") - end - when :key_oid then - case char - when '=' then state = :value - when '0'..'9', '.', ' ' then key << char - else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") - end - when :value then - case char - when '\\' then state = :value_normal_escape - when '"' then state = :value_quoted - when ' ' then state = :value - when '#' then - state = :value_hexstring - value << char - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else - state = :value_normal - value << char - end - when :value_normal then - case char - when '\\' then state = :value_normal_escape - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") - else value << char - end - when :value_normal_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal_escape_hex - hex_buffer = char - else - state = :value_normal - value << char - end - when :value_normal_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_normal - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") - end - when :value_quoted then - case char - when '\\' then state = :value_quoted_escape - when '"' then state = :value_end - else value << char - end - when :value_quoted_escape then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted_escape_hex - hex_buffer = char - else - state = :value_quoted - value << char - end - when :value_quoted_escape_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_quoted - value << "#{hex_buffer}#{char}".to_i(16).chr - else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") - end - when :value_hexstring then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring_hex - value << char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") - end - when :value_hexstring_hex then - case char - when '0'..'9', 'a'..'f', 'A'..'F' then - state = :value_hexstring - value << char - else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") - end - when :value_end then - case char - when ' ' then state = :value_end - when ',' then - state = :key - yield key.string.strip, rstrip_except_escaped(value.string, dn_index) - key = StringIO.new - value = StringIO.new - else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") - end - else raise "Fell out of state machine" - end - end - - # Last pair - raise(MalformedError, 'DN string ended unexpectedly') unless - [:value, :value_normal, :value_hexstring, :value_end].include? state - - yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) - end - - def rstrip_except_escaped(str, dn_index) - str_ends_with_whitespace = str.match(/\s\z/) - - if str_ends_with_whitespace - dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) - - if dn_part_ends_with_escaped_whitespace - dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] - num_chars_to_remove = dn_part_rwhitespace.length - 1 - str = str[0, str.length - num_chars_to_remove] - else - str.rstrip! - end - end - - str - end - - ## - # Returns the DN as an array in the form expected by the constructor. - def to_a - a = [] - self.each_pair { |key, value| a << key << value } unless @dn.empty? - a - end - - ## - # Return the DN as an escaped string. - def to_s - @dn - end - - ## - # Return the DN as an escaped and normalized string. - def to_normalized_s - self.class.new(*to_a).to_s.downcase - end - - # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions - # for DN values. All of the following must be escaped in any normal string - # using a single backslash ('\') as escape. The space character is left - # out here because in a "normalized" string, spaces should only be escaped - # if necessary (i.e. leading or trailing space). - NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze - - # The following must be represented as escaped hex - HEX_ESCAPES = { - "\n" => '\0a', - "\r" => '\0d' - }.freeze - - # Compiled character class regexp using the keys from the above hash, and - # checking for a space or # at the start, or space at the end, of the - # string. - ESCAPE_RE = Regexp.new("(^ |^#| $|[" + - NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + - "])") - - HEX_ESCAPE_RE = Regexp.new("([" + - HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + - "])") - - ## - # Escape a string for use in a DN value - def self.escape(string) - escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } - escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } - end - - private - - def initialize_array(args) - buffer = StringIO.new - - args.each_with_index do |arg, index| - if index.even? # key - buffer << "," if index > 0 - buffer << arg - else # value - buffer << "=" - buffer << self.class.escape(arg) - end - end - - @dn = buffer.string - end - - def initialize_string(arg) - @dn = arg.to_s - end - - ## - # Proxy all other requests to the string object, because a DN is mainly - # used within the library as a string - # rubocop:disable GitlabSecurity/PublicSend - def method_missing(method, *args, &block) - @dn.send(method, *args, &block) - end - - ## - # Redefined to be consistent with redefined `method_missing` behavior - def respond_to?(sym, include_private = false) - @dn.respond_to?(sym, include_private) - end - end - end - end - end - - def perform(start_id, end_id) - return unless migrate? - - ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) - ldap_identities.each do |identity| - identity.extern_uid = Gitlab::Auth::LDAP::DN.new(identity.extern_uid).to_normalized_s - unless identity.save - Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." - end - rescue Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." - end - end - - def migrate? - Identity.table_exists? - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb deleted file mode 100644 index aa4f130538c..00000000000 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This background migration is going to create all `fork_networks` and - # the `fork_network_members` for the roots of fork networks based on the - # existing `forked_project_links`. - # - # When the source of a fork is deleted, we will create the fork with the - # target project as the root. This way, when there are forks of the target - # project, they will be joined into the same fork network. - # - # When the `fork_networks` and memberships for the root projects are created - # the `CreateForkNetworkMembershipsRange` migration is scheduled. This - # migration will create the memberships for all remaining forks-of-forks - class PopulateForkNetworksRange - def perform(start_id, end_id) - create_fork_networks_for_existing_projects(start_id, end_id) - create_fork_networks_for_missing_projects(start_id, end_id) - create_fork_networks_memberships_for_root_projects(start_id, end_id) - - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY - BackgroundMigrationWorker.perform_in( - delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] - ) - end - - def create_fork_networks_for_existing_projects(start_id, end_id) - log("Creating fork networks: #{start_id} - #{end_id}") - ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS - INSERT INTO fork_networks (root_project_id) - SELECT DISTINCT forked_project_links.forked_from_project_id - - FROM forked_project_links - - -- Exclude the forks that are not the first level fork of a project - WHERE NOT EXISTS ( - SELECT true - FROM forked_project_links inner_links - WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id - ) - - /* Exclude the ones that are already created, in case the fork network - was already created for another fork of the project. - */ - AND NOT EXISTS ( - SELECT true - FROM fork_networks - WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id - ) - - -- Only create a fork network for a root project that still exists - AND EXISTS ( - SELECT true - FROM projects - WHERE projects.id = forked_project_links.forked_from_project_id - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - INSERT_NETWORKS - end - - def create_fork_networks_for_missing_projects(start_id, end_id) - log("Creating fork networks with missing root: #{start_id} - #{end_id}") - ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS - INSERT INTO fork_networks (root_project_id) - SELECT DISTINCT forked_project_links.forked_to_project_id - - FROM forked_project_links - - -- Exclude forks that are not the root forks - WHERE NOT EXISTS ( - SELECT true - FROM forked_project_links inner_links - WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id - ) - - /* Exclude the ones that are already created, in case this migration is - re-run - */ - AND NOT EXISTS ( - SELECT true - FROM fork_networks - WHERE forked_project_links.forked_to_project_id = fork_networks.root_project_id - ) - - /* Exclude projects for which the project still exists, those are - Processed in the previous step of this migration - */ - AND NOT EXISTS ( - SELECT true - FROM projects - WHERE projects.id = forked_project_links.forked_from_project_id - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - INSERT_NETWORKS - end - - def create_fork_networks_memberships_for_root_projects(start_id, end_id) - log("Creating memberships for root projects: #{start_id} - #{end_id}") - - ActiveRecord::Base.connection.execute <<~INSERT_ROOT - INSERT INTO fork_network_members (fork_network_id, project_id) - SELECT DISTINCT fork_networks.id, fork_networks.root_project_id - - FROM fork_networks - - /* Joining both on forked_from- and forked_to- so we could create the - memberships for forks for which the source was deleted - */ - INNER JOIN forked_project_links - ON forked_project_links.forked_from_project_id = fork_networks.root_project_id - OR forked_project_links.forked_to_project_id = fork_networks.root_project_id - - WHERE NOT EXISTS ( - SELECT true - FROM fork_network_members - WHERE fork_network_members.project_id = fork_networks.root_project_id - ) - AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} - INSERT_ROOT - end - - def log(message) - Rails.logger.info("#{self.class.name} - #{message}") - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb deleted file mode 100644 index dcac355e1b0..00000000000 --- a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateMergeRequestsLatestMergeRequestDiffId - BATCH_SIZE = 1_000 - - class MergeRequest < ActiveRecord::Base - self.table_name = 'merge_requests' - - include ::EachBatch - end - - def perform(start_id, stop_id) - update = ' - latest_merge_request_diff_id = ( - SELECT MAX(id) - FROM merge_request_diffs - WHERE merge_requests.id = merge_request_diffs.merge_request_id - )'.squish - - MergeRequest - .where(id: start_id..stop_id) - .where(latest_merge_request_diff_id: nil) - .each_batch(of: BATCH_SIZE) do |relation| - - relation.update_all(update) - end - end - end - end -end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 7aeac11df55..cde042c5e0a 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -23,6 +23,11 @@ module Gitlab @root = Entry::Root.new(@config) @root.compose! + + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e + Gitlab::Sentry.track_exception(e, extra: { user: user.inspect, project: project.inspect }) + raise Config::ConfigError, e.message + rescue *rescue_errors => e raise Config::ConfigError, e.message end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 65a6630365d..cf3d261c1cb 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -74,16 +74,16 @@ stages: - cleanup include: - - template: Jobs/Build.gitlab-ci.yml - - template: Jobs/Test.gitlab-ci.yml - - template: Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Deploy.gitlab-ci.yml - - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Management.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml + - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml # Override DAST job to exclude master branch dast: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index dcf8254ef94..108f0119ae1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -246,7 +246,6 @@ rollout 100%: auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} export DATABASE_URL=${DATABASE_URL-$auto_database_url} export TILLER_NAMESPACE=$KUBE_NAMESPACE - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products function get_replicas() { track="${1:-stable}" diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index b9fee2d5731..25ea20e454f 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -1,5 +1,5 @@ # Select image from https://hub.docker.com/_/php/ -image: php:7.1.1 +image: php:latest # Select what we should cache between builds cache: diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 0d742aa282d..e7dacd3a1fc 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -4,6 +4,7 @@ image: ruby:2.3 variables: JEKYLL_ENV: production + LC_ALL: C.UTF-8 before_script: - bundle install diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 8713b833011..0a97a16b83c 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -54,6 +54,7 @@ sast: MAVEN_PATH \ MAVEN_REPO_PATH \ SBT_PATH \ + FAIL_NEVER \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index e0f9eb59924..8f796748199 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -11,6 +11,9 @@ module Gitlab # We have three lifecycle events. # # - before_fork (only in forking processes) + # In forking processes (Unicorn and Puma in multiprocess mode) this + # will be called exactly once, on startup, before the workers are + # forked. This will be called in the parent process. # - worker_start # - before_master_restart (only in forking processes) # diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index 8159f8b8026..4cedab1545c 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -4,6 +4,13 @@ module Gitlab module Config module Loader class Yaml + DataTooLargeError = Class.new(Loader::FormatError) + + include Gitlab::Utils::StrongMemoize + + MAX_YAML_SIZE = 1.megabyte + MAX_YAML_DEPTH = 100 + def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) rescue Psych::Exception => e @@ -11,16 +18,35 @@ module Gitlab end def valid? - @config.is_a?(Hash) + hash? && !too_big? end def load! - unless valid? - raise Loader::FormatError, 'Invalid configuration format' - end + raise DataTooLargeError, 'The parsed YAML is too big' if too_big? + raise Loader::FormatError, 'Invalid configuration format' unless hash? @config.deep_symbolize_keys end + + private + + def hash? + @config.is_a?(Hash) + end + + def too_big? + return false unless Feature.enabled?(:ci_yaml_limit_size, default_enabled: true) + + !deep_size.valid? + end + + def deep_size + strong_memoize(:deep_size) do + Gitlab::Utils::DeepSize.new(@config, + max_size: MAX_YAML_SIZE, + max_depth: MAX_YAML_DEPTH) + end + end end end end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 1ecf4a12db7..0fc145534bf 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -103,6 +103,11 @@ module Gitlab yarn\.lock )\z}x => :frontend, + %r{\A(ee/)?db/} => :database, + %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, + %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, + %r{\Arubocop/cop/migration(/|\.rb)} => :database, + %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|danger|generator_templates|lib|rubocop|scripts)/} => :backend, %r{\A(ee/)?spec/features/} => :test, @@ -112,7 +117,6 @@ module Gitlab %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, - %r{\A(ee/)?db/} => :database, %r{\A(ee/)?qa/} => :qa, # Files that don't fit into any category are marked with :none diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 1455e410d4b..b8d895dee7d 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -158,7 +158,7 @@ module Gitlab Arel::Nodes::Window.new.order(arel_table[column_sym]) ).as('row_id') - count = arel_table.project("COUNT(1)").as('ct') + count = arel_table.where(arel_table[column_sym].gteq(zero_interval)).project("COUNT(1)").as('ct') [column_row, row_id, count] end diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb index 6cf904b2b2a..0bd18fe9622 100644 --- a/lib/gitlab/diff/lines_unfolder.rb +++ b/lib/gitlab/diff/lines_unfolder.rb @@ -54,7 +54,7 @@ module Gitlab def unfold_required? strong_memoize(:unfold_required) do next false unless @diff_file.text? - next false unless @position.unchanged? + next false unless @position.on_text? && @position.unchanged? next false if @diff_file.new_file? || @diff_file.deleted_file? next false unless @position.old_line # Invalid position (MR import scenario) diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index d349c378e53..dfa80eb4a64 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -134,6 +134,10 @@ module Gitlab @line_code ||= diff_file(repository)&.line_code_for_position(self) end + def file_hash + @file_hash ||= Digest::SHA1.hexdigest(file_path) + end + def on_image? position_type == 'image' end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index af3df820422..a1c82ce9afc 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -17,187 +17,13 @@ module Gitlab @paths = paths end - def trace(ab_position) + def trace(old_position) return unless old_diff_refs&.complete? && new_diff_refs&.complete? - return unless ab_position.diff_refs == old_diff_refs + return unless old_position.diff_refs == old_diff_refs - # Suppose we have an MR with source branch `feature` and target branch `master`. - # When the MR was created, the head of `master` was commit A, and the - # head of `feature` was commit B, resulting in the original diff A->B. - # Since creation, `master` was updated to C. - # Now `feature` is being updated to D, and the newly generated MR diff is C->D. - # It is possible that C and D are direct descendants of A and B respectively, - # but this isn't necessarily the case as rebases and merges come into play. - # - # Suppose we have a diff note on the original diff A->B. Now that the MR - # is updated, we need to find out what line in C->D corresponds to the - # line the note was originally created on, so that we can update the diff note's - # records and continue to display it in the right place in the diffs. - # If we cannot find this line in the new diff, this means the diff note is now - # outdated, and we will display that fact to the user. - # - # In the new diff, the file the diff note was originally created on may - # have been renamed, deleted or even created, if the file existed in A and B, - # but was removed in C, and restored in D. - # - # Every diff note stores a Position object that defines a specific location, - # identified by paths and line numbers, within a specific diff, identified - # by start, head and base commit ids. - # - # For diff notes for diff A->B, the position looks like this: - # Position - # start_sha - ID of commit A - # head_sha - ID of commit B - # base_sha - ID of base commit of A and B - # old_path - path as of A (nil if file was newly created) - # new_path - path as of B (nil if file was deleted) - # old_line - line number as of A (nil if file was newly created) - # new_line - line number as of B (nil if file was deleted) - # - # We can easily update `start_sha` and `head_sha` to hold the IDs of - # commits C and D, and can trivially determine `base_sha` based on those, - # but need to find the paths and line numbers as of C and D. - # - # If the file was unchanged or newly created in A->B, the path as of D can be found - # by generating diff B->D ("head to head"), finding the diff file with - # `diff_file.old_path == position.new_path`, and taking `diff_file.new_path`. - # The path as of C can be found by taking diff C->D, finding the diff file - # with that same `new_path` and taking `diff_file.old_path`. - # The line number as of D can be found by using the LineMapper on diff B->D - # and providing the line number as of B. - # The line number as of C can be found by using the LineMapper on diff C->D - # and providing the line number as of D. - # - # If the file was deleted in A->B, the path as of C can be found - # by generating diff A->C ("base to base"), finding the diff file with - # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. - # The path as of D can be found by taking diff C->D, finding the diff file - # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`. - # The line number as of C can be found by using the LineMapper on diff A->C - # and providing the line number as of A. - # The line number as of D can be found by using the LineMapper on diff C->D - # and providing the line number as of C. + strategy = old_position.on_text? ? LineStrategy : ImageStrategy - if ab_position.added? - trace_added_line(ab_position) - elsif ab_position.removed? - trace_removed_line(ab_position) - else # unchanged - trace_unchanged_line(ab_position) - end - end - - private - - def trace_added_line(ab_position) - b_path = ab_position.new_path - b_line = ab_position.new_line - - bd_diff = bd_diffs.diff_file_with_old_path(b_path) - - d_path = bd_diff&.new_path || b_path - d_line = LineMapper.new(bd_diff).old_to_new(b_line) - - if d_line - cd_diff = cd_diffs.diff_file_with_new_path(d_path) - - c_path = cd_diff&.old_path || d_path - c_line = LineMapper.new(cd_diff).new_to_old(d_line) - - if c_line - # If the line is still in D but also in C, it has turned from an - # added line into an unchanged one. - new_position = position(cd_diff, c_line, d_line) - if valid_position?(new_position) - # If the line is still in the MR, we don't treat this as outdated. - { position: new_position, outdated: false } - else - # If the line is no longer in the MR, we unfortunately cannot show - # the current state on the CD diff, so we treat it as outdated. - ac_diff = ac_diffs.diff_file_with_new_path(c_path) - - { position: position(ac_diff, nil, c_line), outdated: true } - end - else - # If the line is still in D and not in C, it is still added. - { position: position(cd_diff, nil, d_line), outdated: false } - end - else - # If the line is no longer in D, it has been removed from the MR. - { position: position(bd_diff, b_line, nil), outdated: true } - end - end - - def trace_removed_line(ab_position) - a_path = ab_position.old_path - a_line = ab_position.old_line - - ac_diff = ac_diffs.diff_file_with_old_path(a_path) - - c_path = ac_diff&.new_path || a_path - c_line = LineMapper.new(ac_diff).old_to_new(a_line) - - if c_line - cd_diff = cd_diffs.diff_file_with_old_path(c_path) - - d_path = cd_diff&.new_path || c_path - d_line = LineMapper.new(cd_diff).old_to_new(c_line) - - if d_line - # If the line is still in C but also in D, it has turned from a - # removed line into an unchanged one. - bd_diff = bd_diffs.diff_file_with_new_path(d_path) - - { position: position(bd_diff, nil, d_line), outdated: true } - else - # If the line is still in C and not in D, it is still removed. - { position: position(cd_diff, c_line, nil), outdated: false } - end - else - # If the line is no longer in C, it has been removed outside of the MR. - { position: position(ac_diff, a_line, nil), outdated: true } - end - end - - def trace_unchanged_line(ab_position) - a_path = ab_position.old_path - a_line = ab_position.old_line - b_path = ab_position.new_path - b_line = ab_position.new_line - - ac_diff = ac_diffs.diff_file_with_old_path(a_path) - - c_path = ac_diff&.new_path || a_path - c_line = LineMapper.new(ac_diff).old_to_new(a_line) - - bd_diff = bd_diffs.diff_file_with_old_path(b_path) - - d_line = LineMapper.new(bd_diff).old_to_new(b_line) - - cd_diff = cd_diffs.diff_file_with_old_path(c_path) - - if c_line && d_line - # If the line is still in C and D, it is still unchanged. - new_position = position(cd_diff, c_line, d_line) - if valid_position?(new_position) - # If the line is still in the MR, we don't treat this as outdated. - { position: new_position, outdated: false } - else - # If the line is no longer in the MR, we unfortunately cannot show - # the current state on the CD diff or any change on the BD diff, - # so we treat it as outdated. - { position: nil, outdated: true } - end - elsif d_line # && !c_line - # If the line is still in D but no longer in C, it has turned from - # an unchanged line into an added one. - # We don't treat this as outdated since the line is still in the MR. - { position: position(cd_diff, nil, d_line), outdated: false } - else # !d_line && (c_line || !c_line) - # If the line is no longer in D, it has turned from an unchanged line - # into a removed one. - { position: position(bd_diff, b_line, nil), outdated: true } - end + strategy.new(self).trace(old_position) end def ac_diffs @@ -216,18 +42,12 @@ module Gitlab @cd_diffs ||= compare(new_diff_refs.start_sha, new_diff_refs.head_sha) end + private + def compare(start_sha, head_sha, straight: false) compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) compare.diffs(paths: paths, expanded: true) end - - def position(diff_file, old_line, new_line) - Position.new(diff_file: diff_file, old_line: old_line, new_line: new_line) - end - - def valid_position?(position) - !!position.diff_line(project.repository) - end end end end diff --git a/lib/gitlab/diff/position_tracer/base_strategy.rb b/lib/gitlab/diff/position_tracer/base_strategy.rb new file mode 100644 index 00000000000..65049daabf4 --- /dev/null +++ b/lib/gitlab/diff/position_tracer/base_strategy.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class BaseStrategy + attr_reader :tracer + + delegate \ + :project, + :ac_diffs, + :bd_diffs, + :cd_diffs, + to: :tracer + + def initialize(tracer) + @tracer = tracer + end + + def trace(position) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer/image_strategy.rb b/lib/gitlab/diff/position_tracer/image_strategy.rb new file mode 100644 index 00000000000..79244a17951 --- /dev/null +++ b/lib/gitlab/diff/position_tracer/image_strategy.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class ImageStrategy < BaseStrategy + def trace(position) + b_path = position.new_path + + # If file exists in B->D (e.g. updated, renamed, removed), let the + # note become outdated. + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + return { position: new_position(position, bd_diff), outdated: true } if bd_diff + + # If file still exists in the new diff, update the position. + cd_diff = cd_diffs.diff_file_with_new_path(bd_diff&.new_path || b_path) + + return { position: new_position(position, cd_diff), outdated: false } if cd_diff + + # If file exists in A->C (e.g. rebased and same changes were present + # in target branch), let the note become outdated. + ac_diff = ac_diffs.diff_file_with_old_path(position.old_path) + + return { position: new_position(position, ac_diff), outdated: true } if ac_diff + + # If ever there's a case that the file no longer exists in any diff, + # don't set a change position and let the note become outdated. + # + # This should never happen given the file should exist in one of the + # diffs above. + { outdated: true } + end + + private + + def new_position(position, diff_file) + Position.new( + diff_file: diff_file, + x: position.x, + y: position.y, + width: position.width, + height: position.height, + position_type: position.position_type + ) + end + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb new file mode 100644 index 00000000000..8db0fc6f963 --- /dev/null +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -0,0 +1,201 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class LineStrategy < BaseStrategy + def trace(position) + # Suppose we have an MR with source branch `feature` and target branch `master`. + # When the MR was created, the head of `master` was commit A, and the + # head of `feature` was commit B, resulting in the original diff A->B. + # Since creation, `master` was updated to C. + # Now `feature` is being updated to D, and the newly generated MR diff is C->D. + # It is possible that C and D are direct descendants of A and B respectively, + # but this isn't necessarily the case as rebases and merges come into play. + # + # Suppose we have a diff note on the original diff A->B. Now that the MR + # is updated, we need to find out what line in C->D corresponds to the + # line the note was originally created on, so that we can update the diff note's + # records and continue to display it in the right place in the diffs. + # If we cannot find this line in the new diff, this means the diff note is now + # outdated, and we will display that fact to the user. + # + # In the new diff, the file the diff note was originally created on may + # have been renamed, deleted or even created, if the file existed in A and B, + # but was removed in C, and restored in D. + # + # Every diff note stores a Position object that defines a specific location, + # identified by paths and line numbers, within a specific diff, identified + # by start, head and base commit ids. + # + # For diff notes for diff A->B, the position looks like this: + # Position + # start_sha - ID of commit A + # head_sha - ID of commit B + # base_sha - ID of base commit of A and B + # old_path - path as of A (nil if file was newly created) + # new_path - path as of B (nil if file was deleted) + # old_line - line number as of A (nil if file was newly created) + # new_line - line number as of B (nil if file was deleted) + # + # We can easily update `start_sha` and `head_sha` to hold the IDs of + # commits C and D, and can trivially determine `base_sha` based on those, + # but need to find the paths and line numbers as of C and D. + # + # If the file was unchanged or newly created in A->B, the path as of D can be found + # by generating diff B->D ("head to head"), finding the diff file with + # `diff_file.old_path == position.new_path`, and taking `diff_file.new_path`. + # The path as of C can be found by taking diff C->D, finding the diff file + # with that same `new_path` and taking `diff_file.old_path`. + # The line number as of D can be found by using the LineMapper on diff B->D + # and providing the line number as of B. + # The line number as of C can be found by using the LineMapper on diff C->D + # and providing the line number as of D. + # + # If the file was deleted in A->B, the path as of C can be found + # by generating diff A->C ("base to base"), finding the diff file with + # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. + # The path as of D can be found by taking diff C->D, finding the diff file + # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`. + # The line number as of C can be found by using the LineMapper on diff A->C + # and providing the line number as of A. + # The line number as of D can be found by using the LineMapper on diff C->D + # and providing the line number as of C. + + if position.added? + trace_added_line(position) + elsif position.removed? + trace_removed_line(position) + else # unchanged + trace_unchanged_line(position) + end + end + + private + + def trace_added_line(position) + b_path = position.new_path + b_line = position.new_line + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_path = bd_diff&.new_path || b_path + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + if d_line + cd_diff = cd_diffs.diff_file_with_new_path(d_path) + + c_path = cd_diff&.old_path || d_path + c_line = LineMapper.new(cd_diff).new_to_old(d_line) + + if c_line + # If the line is still in D but also in C, it has turned from an + # added line into an unchanged one. + new_position = new_position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff, so we treat it as outdated. + ac_diff = ac_diffs.diff_file_with_new_path(c_path) + + { position: new_position(ac_diff, nil, c_line), outdated: true } + end + else + # If the line is still in D and not in C, it is still added. + { position: new_position(cd_diff, nil, d_line), outdated: false } + end + else + # If the line is no longer in D, it has been removed from the MR. + { position: new_position(bd_diff, b_line, nil), outdated: true } + end + end + + def trace_removed_line(position) + a_path = position.old_path + a_line = position.old_line + + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) + + if c_line + cd_diff = cd_diffs.diff_file_with_old_path(c_path) + + d_path = cd_diff&.new_path || c_path + d_line = LineMapper.new(cd_diff).old_to_new(c_line) + + if d_line + # If the line is still in C but also in D, it has turned from a + # removed line into an unchanged one. + bd_diff = bd_diffs.diff_file_with_new_path(d_path) + + { position: new_position(bd_diff, nil, d_line), outdated: true } + else + # If the line is still in C and not in D, it is still removed. + { position: new_position(cd_diff, c_line, nil), outdated: false } + end + else + # If the line is no longer in C, it has been removed outside of the MR. + { position: new_position(ac_diff, a_line, nil), outdated: true } + end + end + + def trace_unchanged_line(position) + a_path = position.old_path + a_line = position.old_line + b_path = position.new_path + b_line = position.new_line + + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + cd_diff = cd_diffs.diff_file_with_old_path(c_path) + + if c_line && d_line + # If the line is still in C and D, it is still unchanged. + new_position = new_position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff or any change on the BD diff, + # so we treat it as outdated. + { position: nil, outdated: true } + end + elsif d_line # && !c_line + # If the line is still in D but no longer in C, it has turned from + # an unchanged line into an added one. + # We don't treat this as outdated since the line is still in the MR. + { position: new_position(cd_diff, nil, d_line), outdated: false } + else # !d_line && (c_line || !c_line) + # If the line is no longer in D, it has turned from an unchanged line + # into a removed one. + { position: new_position(bd_diff, b_line, nil), outdated: true } + end + end + + def new_position(diff_file, old_line, new_line) + Position.new( + diff_file: diff_file, + old_line: old_line, + new_line: new_line + ) + end + + def valid_position?(position) + !!position.diff_line(project.repository) + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 19b6aab1c4f..060a29be782 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -536,9 +536,9 @@ module Gitlab tags.find { |tag| tag.name == name } end - def merge_to_ref(user, source_sha, branch, target_ref, message) + def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) wrapped_gitaly_errors do - gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message) + gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) end end diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 11ee4ebda4b..86c9f33d82a 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -11,10 +11,11 @@ module Gitlab module Blob module ClassMethods extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged override :tree_entry def tree_entry(repository, sha, path, limit) - if Feature.enabled?(:rugged_tree_entry) + if use_rugged?(repository, :rugged_tree_entry) rugged_tree_entry(repository, sha, path, limit) else super diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index bce4fa14fb4..971a33b2e99 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -12,6 +12,7 @@ module Gitlab module Commit module ClassMethods extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged def rugged_find(repo, commit_id) obj = repo.rev_parse_target(commit_id) @@ -34,7 +35,7 @@ module Gitlab override :find_commit def find_commit(repo, commit_id) - if Feature.enabled?(:rugged_find_commit) + if use_rugged?(repo, :rugged_find_commit) rugged_find(repo, commit_id) else super @@ -43,7 +44,7 @@ module Gitlab override :batch_by_oid def batch_by_oid(repo, oids) - if Feature.enabled?(:rugged_list_commits_by_oid) + if use_rugged?(repo, :rugged_list_commits_by_oid) rugged_batch_by_oid(repo, oids) else super @@ -52,6 +53,7 @@ module Gitlab end extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged override :init_commit def init_commit(raw_commit) @@ -65,7 +67,7 @@ module Gitlab override :commit_tree_entry def commit_tree_entry(path) - if Feature.enabled?(:rugged_commit_tree_entry) + if use_rugged?(@repository, :rugged_commit_tree_entry) rugged_tree_entry(path) else super diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index e91b0ddcd31..9268abdfed9 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -11,6 +11,7 @@ module Gitlab module RuggedImpl module Repository extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid).freeze @@ -46,7 +47,7 @@ module Gitlab override :ancestor? def ancestor?(from, to) - if Feature.enabled?(:rugged_commit_is_ancestor) + if use_rugged?(self, :rugged_commit_is_ancestor) rugged_is_ancestor?(from, to) else super diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 9c37bb01961..f3721a3f1b7 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -11,10 +11,11 @@ module Gitlab module Tree module ClassMethods extend ::Gitlab::Utils::Override + include Gitlab::Git::RuggedImpl::UseRugged override :tree_entries def tree_entries(repository, sha, path, recursive) - if Feature.enabled?(:rugged_tree_entries) + if use_rugged?(repository, :rugged_tree_entries) tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) else super diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb new file mode 100644 index 00000000000..99091b03cd1 --- /dev/null +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module RuggedImpl + module UseRugged + def use_rugged?(repo, feature_key) + feature = Feature.get(feature_key) + return feature.enabled? if Feature.persisted?(feature) + + Gitlab::GitalyClient.can_use_disk?(repo.storage) + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 47976389af6..cf0157269a8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -30,14 +30,10 @@ module Gitlab SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze + GITALY_METADATA_FILENAME = '.gitaly-metadata' MUTEX = Mutex.new - define_histogram :gitaly_controller_action_duration_seconds do - docstring "Gitaly endpoint histogram by controller and action combination" - base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil) - end - def self.stub(name, storage) MUTEX.synchronize do @stubs ||= {} @@ -161,10 +157,6 @@ module Gitlab # Keep track, separately, for the performance bar self.query_time += duration - gitaly_controller_action_duration_seconds.observe( - current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), - duration) - if peek_enabled? add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc, backtrace: Gitlab::Profiler.clean_backtrace(caller)) @@ -387,6 +379,45 @@ module Gitlab 0 end + def self.storage_metadata_file_path(storage) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join( + Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME + ) + end + end + + def self.can_use_disk?(storage) + cached_value = MUTEX.synchronize do + @can_use_disk ||= {} + @can_use_disk[storage] + end + + return cached_value unless cached_value.nil? + + gitaly_filesystem_id = filesystem_id(storage) + direct_filesystem_id = filesystem_id_from_disk(storage) + + MUTEX.synchronize do + @can_use_disk[storage] = gitaly_filesystem_id.present? && + gitaly_filesystem_id == direct_filesystem_id + end + end + + def self.filesystem_id(storage) + response = Gitlab::GitalyClient::ServerService.new(storage).info + storage_status = response.storage_statuses.find { |status| status.storage_name == storage } + storage_status.filesystem_id + end + + def self.filesystem_id_from_disk(storage) + metadata_file = File.read(storage_metadata_file_path(storage)) + metadata_hash = JSON.parse(metadata_file) + metadata_hash['gitaly_filesystem_id'] + rescue Errno::ENOENT, JSON::ParserError + nil + end + def self.timeout(timeout_name) Gitlab::CurrentSettings.current_application_settings[timeout_name] end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index b42e6cbad8d..783c2ff0915 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -100,14 +100,15 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha, branch, target_ref, message) + def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, branch: encode_binary(branch), target_ref: encode_binary(target_ref), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - message: encode_binary(message) + message: encode_binary(message), + first_parent_ref: encode_binary(first_parent_ref) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request) diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 92917028851..41ec8741eb1 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -38,6 +38,11 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url end + + # Flag controls a GFM feature used across many routes. + # Pushing the flag from one place simplifies control + # and facilitates easy removal. + push_frontend_feature_flag(:gfm_embedded_metrics) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb index 8a59e83974f..74c04e5380e 100644 --- a/lib/gitlab/graphql.rb +++ b/lib/gitlab/graphql.rb @@ -3,9 +3,5 @@ module Gitlab module Graphql StandardGraphqlError = Class.new(StandardError) - - def self.enabled? - Feature.enabled?(:graphql, default_enabled: true) - end end end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb index f8d0208e275..e83b567308b 100644 --- a/lib/gitlab/graphql/authorize.rb +++ b/lib/gitlab/graphql/authorize.rb @@ -8,7 +8,7 @@ module Gitlab extend ActiveSupport::Concern def self.use(schema_definition) - schema_definition.instrument(:field, Instrumentation.new, after_built_ins: true) + schema_definition.instrument(:field, Gitlab::Graphql::Authorize::Instrumentation.new, after_built_ins: true) end end end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index 619ce100421..3b5dde2fde5 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -39,6 +39,8 @@ module Gitlab type = node_type_for_basic_connection(type) end + type = type.unwrap if type.kind.non_null? + Array.wrap(type.metadata[:authorize]) end diff --git a/lib/gitlab/graphql/calls_gitaly.rb b/lib/gitlab/graphql/calls_gitaly.rb new file mode 100644 index 00000000000..40cd74a34f2 --- /dev/null +++ b/lib/gitlab/graphql/calls_gitaly.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + # Wraps the field resolution to count Gitaly calls before and after. + # Raises an error if the field calls Gitaly but hadn't declared such. + module CallsGitaly + extend ActiveSupport::Concern + + def self.use(schema_definition) + schema_definition.instrument(:field, Gitlab::Graphql::CallsGitaly::Instrumentation.new, after_built_ins: true) + end + end + end +end diff --git a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb new file mode 100644 index 00000000000..fbd5e348c7d --- /dev/null +++ b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module CallsGitaly + class Instrumentation + # Check if any `calls_gitaly: true` declarations need to be added + # Do nothing if a constant complexity was provided + def instrument(_type, field) + type_object = field.metadata[:type_class] + return field unless type_object.respond_to?(:calls_gitaly?) + return field if type_object.constant_complexity? || type_object.calls_gitaly? + + old_resolver_proc = field.resolve_proc + + gitaly_wrapped_resolve = -> (typed_object, args, ctx) do + previous_gitaly_call_count = Gitlab::GitalyClient.get_request_count + result = old_resolver_proc.call(typed_object, args, ctx) + current_gitaly_call_count = Gitlab::GitalyClient.get_request_count + calls_gitaly_check(type_object, current_gitaly_call_count - previous_gitaly_call_count) + result + end + + field.redefine do + resolve(gitaly_wrapped_resolve) + end + end + + def calls_gitaly_check(type_object, calls) + return if calls < 1 + + # Will inform you if there needs to be `calls_gitaly: true` as a kwarg in the field declaration + # if there is at least 1 Gitaly call involved with the field resolution. + error = RuntimeError.new("Gitaly is called for field '#{type_object.name}' on #{type_object.owner.try(:name)} - please either specify a constant complexity or add `calls_gitaly: true` to the field declaration") + Gitlab::Sentry.track_exception(error) + end + end + end + end +end diff --git a/lib/gitlab/graphql/find_argument_in_parent.rb b/lib/gitlab/graphql/find_argument_in_parent.rb new file mode 100644 index 00000000000..1f83f8fce7a --- /dev/null +++ b/lib/gitlab/graphql/find_argument_in_parent.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module FindArgumentInParent + # Searches up the GraphQL AST and returns the first matching argument + # passed to a node + def self.find(parent, argument, limit_depth: nil) + argument = argument.to_s.camelize(:lower).to_sym + depth = 0 + + while parent.respond_to?(:parent) + args = node_args(parent) + return args[argument] if args.key?(argument) + + depth += 1 + return if limit_depth && depth >= limit_depth + + parent = parent.parent + end + end + + class << self + private + + def node_args(node) + node.irep_node.arguments + end + end + end + end +end diff --git a/lib/gitlab/graphql/mount_mutation.rb b/lib/gitlab/graphql/mount_mutation.rb index 9048967d4e1..b10e963170a 100644 --- a/lib/gitlab/graphql/mount_mutation.rb +++ b/lib/gitlab/graphql/mount_mutation.rb @@ -6,11 +6,12 @@ module Gitlab extend ActiveSupport::Concern class_methods do - def mount_mutation(mutation_class) + def mount_mutation(mutation_class, **custom_kwargs) # Using an underscored field name symbol will make `graphql-ruby` # standardize the field name field mutation_class.graphql_name.underscore.to_sym, - mutation: mutation_class + mutation: mutation_class, + **custom_kwargs end end end diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index db2b4dde244..58bce613a98 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -10,9 +10,9 @@ module Gitlab RedirectionTooDeep = Class.new(StandardError) HTTP_ERRORS = [ - SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, - Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, - Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, + SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, + Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, + Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep ].freeze diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index a0fb051e806..01437c67fa9 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -160,6 +160,7 @@ excluded_attributes: - :milestone_id - :ref_fetched - :merge_jid + - :rebase_jid - :latest_merge_request_diff_id award_emoji: - :awardable_id diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index 351d15605e0..be73bcd5506 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -2,7 +2,7 @@ module Gitlab module IssuableMetadata - def issuable_meta_data(issuable_collection, collection_type) + def issuable_meta_data(issuable_collection, collection_type, user = nil) # ActiveRecord uses Object#extend for null relations. if !(issuable_collection.singleton_class < ActiveRecord::NullRelation) && issuable_collection.respond_to?(:limit_value) && @@ -23,7 +23,7 @@ module Gitlab issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) issuable_merge_requests_count = if collection_type == 'Issue' - ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids, user) else [] end diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 746786b5a66..fdab6b512ea 100644 --- a/lib/gitlab/legacy_github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -10,6 +10,7 @@ module Gitlab name: raw_data.name, description: raw_data.body, created_at: raw_data.created_at, + released_at: raw_data.published_at, updated_at: raw_data.created_at } end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 17eacbd21d8..eef802caabb 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,6 +6,12 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler + def initialize(interval) + metrics[:process_start_time_seconds].set(labels.merge(worker_label), Time.now.to_i) + + super + end + def metrics @metrics ||= init_metrics end @@ -47,7 +53,6 @@ module Gitlab metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) - metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time) set_memory_usage_metrics sample_gc diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 34de40ca72f..5c2f07b95e2 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -31,14 +31,6 @@ module Gitlab match[1].to_i end - - def self.process_start_time - fields = File.read('/proc/self/stat').split - - # fields[21] is linux proc stat field "(22) starttime". - # The value is expressed in clock ticks, divide by clock ticks for seconds. - ( fields[21].to_i || 0 ) / clk_tck - end else def self.memory_usage 0.0 @@ -51,10 +43,6 @@ module Gitlab def self.max_open_file_descriptors 0 end - - def self.process_start_time - 0 - end end def self.cpu_time diff --git a/lib/gitlab/namespaced_session_store.rb b/lib/gitlab/namespaced_session_store.rb index 34520078bfb..f0f24c081c3 100644 --- a/lib/gitlab/namespaced_session_store.rb +++ b/lib/gitlab/namespaced_session_store.rb @@ -4,19 +4,24 @@ module Gitlab class NamespacedSessionStore delegate :[], :[]=, to: :store - def initialize(key) + def initialize(key, session = Session.current) @key = key + @session = session end def initiated? - !Session.current.nil? + !session.nil? end def store - return unless Session.current + return unless session - Session.current[@key] ||= {} - Session.current[@key] + session[@key] ||= {} + session[@key] end + + private + + attr_reader :session end end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 4b0c7b5c7f8..07439d8e011 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -3,7 +3,8 @@ module Gitlab module PerformanceBar ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze - EXPIRY_TIME = 5.minutes + EXPIRY_TIME_L1_CACHE = 1.minute + EXPIRY_TIME_L2_CACHE = 5.minutes def self.enabled?(user = nil) return true if Rails.env.development? @@ -19,20 +20,31 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def self.allowed_user_ids - Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do - group = Group.find_by_id(allowed_group_id) + l1_cache_backend.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME_L1_CACHE) do + l2_cache_backend.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME_L2_CACHE) do + group = Group.find_by_id(allowed_group_id) - if group - GroupMembersFinder.new(group).execute.pluck(:user_id) - else - [] + if group + GroupMembersFinder.new(group).execute.pluck(:user_id) + else + [] + end end end end # rubocop: enable CodeReuse/ActiveRecord def self.expire_allowed_user_ids_cache - Rails.cache.delete(ALLOWED_USER_IDS_KEY) + l1_cache_backend.delete(ALLOWED_USER_IDS_KEY) + l2_cache_backend.delete(ALLOWED_USER_IDS_KEY) + end + + def self.l1_cache_backend + Gitlab::ThreadMemoryCache.cache_backend + end + + def self.l2_cache_backend + Rails.cache end end end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb new file mode 100644 index 00000000000..2d997760c46 --- /dev/null +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Adapted from https://github.com/peek/peek/blob/master/lib/peek/adapters/redis.rb +module Gitlab + module PerformanceBar + module RedisAdapterWhenPeekEnabled + def save + super unless ::Peek.request_id.blank? + end + end + end +end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 572c55efcc2..f7f89d4e897 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -146,8 +146,8 @@ module Gitlab @updates[:todo_event] = 'add' end - desc _('Mark todo as done') - explanation _('Marks todo as done.') + desc _('Mark to do as done') + explanation _('Marks to do as done.') types Issuable condition do quick_action_target.persisted? && diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index cfbe7f59a83..fa09ecbdf30 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -28,7 +28,7 @@ module Gitlab @binary_data = opts.fetch(:data, nil) @per_page = opts.fetch(:per_page, 20) @project = opts.fetch(:project, nil) - # Some caller (e.g. Elasticsearch) does not have project object, + # Some callers (e.g. Elasticsearch) do not have the Project object, # yet they can trigger many calls in one go, # causing duplicated queries. # Allow those to just pass project_id instead. diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 583a970bf4e..0f890a12134 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -53,14 +53,14 @@ module Gitlab self.num_running(job_ids).zero? end - # Returns true if the given job is running + # Returns true if the given job is running or enqueued. # # job_id - The Sidekiq job ID to check. def self.running?(job_id) num_running([job_id]) > 0 end - # Returns the number of jobs that are running. + # Returns the number of jobs that are running or enqueued. # # job_ids - The Sidekiq job IDs to check. def self.num_running(job_ids) @@ -81,7 +81,7 @@ module Gitlab # job_ids - The Sidekiq job IDs to check. # # Returns an array of true or false indicating job completion. - # true = job is still running + # true = job is still running or enqueued # false = job completed def self.job_status(job_ids) keys = job_ids.map { |jid| key_for(jid) } diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index fd108b4c124..f6edbfced7f 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -9,14 +9,16 @@ module Gitlab REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/.freeze class_methods do - def fuzzy_search(query, columns) - matches = columns.map { |col| fuzzy_arel_match(col, query) }.compact.reduce(:or) + def fuzzy_search(query, columns, use_minimum_char_limit: true) + matches = columns.map do |col| + fuzzy_arel_match(col, query, use_minimum_char_limit: use_minimum_char_limit) + end.compact.reduce(:or) where(matches) end - def to_pattern(query) - if partial_matching?(query) + def to_pattern(query, use_minimum_char_limit: true) + if partial_matching?(query, use_minimum_char_limit: use_minimum_char_limit) "%#{sanitize_sql_like(query)}%" else sanitize_sql_like(query) @@ -27,7 +29,9 @@ module Gitlab MIN_CHARS_FOR_PARTIAL_MATCHING end - def partial_matching?(query) + def partial_matching?(query, use_minimum_char_limit: true) + return true unless use_minimum_char_limit + query.length >= min_chars_for_partial_matching end @@ -35,14 +39,14 @@ module Gitlab # query - The text to search for. # lower_exact_match - When set to `true` we'll fall back to using # `LOWER(column) = query` instead of using `ILIKE`. - def fuzzy_arel_match(column, query, lower_exact_match: false) + def fuzzy_arel_match(column, query, lower_exact_match: false, use_minimum_char_limit: true) query = query.squish return unless query.present? - words = select_fuzzy_words(query) + words = select_fuzzy_words(query, use_minimum_char_limit: use_minimum_char_limit) if words.any? - words.map { |word| arel_table[column].matches(to_pattern(word)) }.reduce(:and) + words.map { |word| arel_table[column].matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) else # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole @@ -56,7 +60,7 @@ module Gitlab end end - def select_fuzzy_words(query) + def select_fuzzy_words(query, use_minimum_char_limit: true) quoted_words = query.scan(REGEX_QUOTED_WORD) query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } @@ -67,7 +71,7 @@ module Gitlab words.concat(quoted_words) - words.select { |word| partial_matching?(word) } + words.select { |word| partial_matching?(word, use_minimum_char_limit: use_minimum_char_limit) } end end end diff --git a/lib/gitlab/thread_memory_cache.rb b/lib/gitlab/thread_memory_cache.rb new file mode 100644 index 00000000000..7f363dc7feb --- /dev/null +++ b/lib/gitlab/thread_memory_cache.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + class ThreadMemoryCache + THREAD_KEY = :thread_memory_cache + + def self.cache_backend + # Note ActiveSupport::Cache::MemoryStore is thread-safe. Since + # each backend is local per thread we probably don't need to worry + # about synchronizing access, but this is a drop-in replacement + # for ActiveSupport::Cache::RedisStore. + Thread.current[THREAD_KEY] ||= ActiveSupport::Cache::MemoryStore.new + end + end +end diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb deleted file mode 100644 index ede60c9ab1d..00000000000 --- a/lib/gitlab/user_extractor.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -# This class extracts all users found in a piece of text by the username or the -# email address - -module Gitlab - class UserExtractor - # Not using `Devise.email_regexp` to filter out any chars that an email - # does not end with and not pinning the email to a start of end of a string. - EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/.freeze - USERNAME_REGEXP = User.reference_pattern - - def initialize(text) - # EE passes an Array to `text` in a few places, so we want to support both - # here. - @text = Array(text).join(' ') - end - - def users - return User.none unless @text.present? - return User.none if references.empty? - - @users ||= User.from_union(union_relations) - end - - def usernames - matches[:usernames] - end - - def emails - matches[:emails] - end - - def references - @references ||= matches.values.flatten - end - - def matches - @matches ||= { - emails: @text.scan(EMAIL_REGEXP).flatten.uniq, - usernames: @text.scan(USERNAME_REGEXP).flatten.uniq - } - end - - private - - def union_relations - relations = [] - - relations << User.by_any_email(emails) if emails.any? - relations << User.by_username(usernames) if usernames.any? - - relations - end - end -end diff --git a/lib/gitlab/utils/deep_size.rb b/lib/gitlab/utils/deep_size.rb new file mode 100644 index 00000000000..562cf09e249 --- /dev/null +++ b/lib/gitlab/utils/deep_size.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require 'objspace' + +module Gitlab + module Utils + class DeepSize + Error = Class.new(StandardError) + TooMuchDataError = Class.new(Error) + + DEFAULT_MAX_SIZE = 1.megabyte + DEFAULT_MAX_DEPTH = 100 + + def initialize(root, max_size: DEFAULT_MAX_SIZE, max_depth: DEFAULT_MAX_DEPTH) + @root = root + @max_size = max_size + @max_depth = max_depth + @size = 0 + @depth = 0 + + evaluate + end + + def valid? + !too_big? && !too_deep? + end + + private + + def evaluate + add_object(@root) + rescue Error + # NOOP + end + + def too_big? + @size > @max_size + end + + def too_deep? + @depth > @max_depth + end + + def add_object(object) + @size += ObjectSpace.memsize_of(object) + raise TooMuchDataError if @size > @max_size + + add_array(object) if object.is_a?(Array) + add_hash(object) if object.is_a?(Hash) + end + + def add_array(object) + with_nesting do + object.each do |n| + add_object(n) + end + end + end + + def add_hash(object) + with_nesting do + object.each do |key, value| + add_object(key) + add_object(value) + end + end + end + + def with_nesting + @depth += 1 + raise TooMuchDataError if too_deep? + + yield + + @depth -= 1 + end + end + end +end diff --git a/lib/peek/views/redis.rb b/lib/peek/views/redis.rb new file mode 100644 index 00000000000..73de8672fa4 --- /dev/null +++ b/lib/peek/views/redis.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'redis' +require 'peek-redis' + +module Gitlab + module Peek + module RedisInstrumented + def call(*args, &block) + start = Time.now + super(*args, &block) + ensure + duration = (Time.now - start) + add_call_details(duration, args) + end + + private + + def add_call_details(duration, args) + # redis-rb passes an array (e.g. [:get, key]) + return unless args.length == 1 + + detail_store << { + cmd: args.first, + duration: duration, + backtrace: Gitlab::Profiler.clean_backtrace(caller) + } + end + + def detail_store + ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] + end + end + end +end + +module Peek + module Views + module RedisDetailed + REDACTED_MARKER = "<redacted>" + + def results + super.merge(details: details) + end + + def details + detail_store + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def detail_store + ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] + end + + def format_call_details(call) + call.merge(cmd: format_command(call[:cmd]), + duration: (call[:duration] * 1000).round(3)) + end + + def format_command(cmd) + if cmd.length >= 2 && cmd.first =~ /^auth$/i + cmd[-1] = REDACTED_MARKER + # Scrub out the value of the SET calls to avoid binary + # data or large data from spilling into the view + elsif cmd.length >= 3 && cmd.first =~ /set/i + cmd[2..-1] = REDACTED_MARKER + end + + cmd.join(' ') + end + end + end +end + +class Redis::Client + prepend Gitlab::Peek::RedisInstrumented +end + +module Peek + module Views + class Redis < View + prepend Peek::Views::RedisDetailed + end + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index f69d204c579..cda88c130bb 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,23 +1,9 @@ desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do - require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') - require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') - require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') - require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') - require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') - require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') - require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb') require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb') - NamespacesProjectsPathLowerIndexes.new.up - AddUsersLowerUsernameEmailIndexes.new.up - AddLowerPathIndexToRoutes.new.up - IndexRoutesPathForLike.new.up - AddLowerPathIndexToRedirectRoutes.new.up - IndexRedirectRoutesPathForLike.new.up - AddIndexOnNamespacesLowerName.new.up UsersNameLowerIndex.new.up ProjectNameLowerIndex.new.up AddPathIndexToRedirectRoutes.new.up diff --git a/lib/tasks/yarn.rake b/lib/tasks/yarn.rake index 2ac88a039e7..32061ad4a57 100644 --- a/lib/tasks/yarn.rake +++ b/lib/tasks/yarn.rake @@ -24,7 +24,7 @@ namespace :yarn do desc 'Install Node dependencies with Yarn' task install: ['yarn:available'] do - unless system('yarn install --pure-lockfile --ignore-engines') + unless system('yarn install --pure-lockfile --ignore-engines --prefer-offline') abort 'Error: Unable to install node modules.'.color(:red) end end |