diff options
| author | Jose Ivan Vargas <jvargas@gitlab.com> | 2018-02-05 15:16:41 -0600 |
|---|---|---|
| committer | Jose Ivan Vargas <jvargas@gitlab.com> | 2018-02-05 15:16:41 -0600 |
| commit | 46ae03628de47d1bef2683a3a5fe4963b3df7d52 (patch) | |
| tree | aca69b4acab10d5699a6315556de7e7530e65dbe /lib/gitlab | |
| parent | e6016d0bc2b640801914369e25e1a3639d3e50eb (diff) | |
| parent | 2150ed4094ddb67d7b403cd56360700c80e7d928 (diff) | |
| download | gitlab-ce-46ae03628de47d1bef2683a3a5fe4963b3df7d52.tar.gz | |
Merge branch 'master' into jivl-update-katex
Diffstat (limited to 'lib/gitlab')
175 files changed, 4471 insertions, 1353 deletions
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index cead1c7eacd..ee7f4be6b9f 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -6,8 +6,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - extend Gitlab::CurrentSettings - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' @@ -33,9 +31,9 @@ module Gitlab def self.plantuml_setup Asciidoctor::PlantUml.configure do |conf| - conf.url = current_application_settings.plantuml_url - conf.svg_enable = current_application_settings.plantuml_enabled - conf.png_enable = current_application_settings.plantuml_enabled + conf.url = Gitlab::CurrentSettings.plantuml_url + conf.svg_enable = Gitlab::CurrentSettings.plantuml_enabled + conf.png_enable = Gitlab::CurrentSettings.plantuml_enabled conf.txt_enable = false end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 65d7fd3ec70..05932378173 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -14,8 +14,6 @@ module Gitlab DEFAULT_SCOPES = [:api].freeze class << self - include Gitlab::CurrentSettings - def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -57,7 +55,7 @@ module Gitlab if user.nil? || user.ldap_user? # Second chance - try LDAP authentication Gitlab::LDAP::Authentication.login(login, password) - elsif current_application_settings.password_authentication_enabled_for_git? + elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? user if user.active? && user.valid_password?(password) end end @@ -87,7 +85,7 @@ module Gitlab private def authenticate_using_internal_or_ldap_password? - current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? end def service_request_check(login, password, project) diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb new file mode 100644 index 00000000000..dae03a179e4 --- /dev/null +++ b/lib/gitlab/auth/blocked_user_tracker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true +module Gitlab + module Auth + class BlockedUserTracker + ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters' + + def self.log_if_user_blocked(env) + message = env.dig('warden.options', :message) + + # Devise calls User#active_for_authentication? on the User model and then + # throws an exception to Warden with User#inactive_message: + # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8 + # + # Since Warden doesn't pass the user record to the failure handler, we + # need to do a database lookup with the username. We can limit the + # lookups to happen when the user was blocked by checking the inactive + # message passed along by Warden. + return unless message == User::BLOCKED_MESSAGE + + login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') + + return unless login.present? + + user = User.by_login(login) + + return unless user&.blocked? + + Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{env['REMOTE_ADDR']}") + SystemHooksService.new.execute_hooks_for(user, :failed_login) + + true + rescue TypeError + end + end + end +end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index b4114a3ac96..cf02030c577 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -96,9 +96,7 @@ module Gitlab end def ensure_action_dispatch_request(request) - return request if request.is_a?(ActionDispatch::Request) - - ActionDispatch::Request.new(request.env) + ActionDispatch::Request.new(request.env.dup) end def current_request diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb new file mode 100644 index 00000000000..7bffffec94d --- /dev/null +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation +# rubocop:disable Metrics/LineLength + +module Gitlab + module BackgroundMigration + class AddMergeRequestDiffCommitsCount + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + def perform(start_id, stop_id) + Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") + + update = ' + commits_count = ( + SELECT count(*) + FROM merge_request_diff_commits + WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id + )'.squish + + MergeRequestDiff.where(id: start_id..stop_id).update_all(update) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb new file mode 100644 index 00000000000..de622f657b2 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentTypeChange + include Database::MigrationHelpers + + RESCHEDULE_DELAY = 10.minutes + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(:issues, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + 'CleanupConcurrentTypeChange', + [table, old_column, new_column] + ) + else + cleanup_concurrent_column_type_change(table, old_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column.rb b/lib/gitlab/background_migration/copy_column.rb new file mode 100644 index 00000000000..ef70f37d5eb --- /dev/null +++ b/lib/gitlab/background_migration/copy_column.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # CopyColumn is a simple (reusable) background migration that can be used to + # update the value of a column based on the value of another column in the + # same table. + # + # For this background migration to work the table that is migrated _has_ to + # have an `id` column as the primary key. + class CopyColumn + # table - The name of the table that contains the columns. + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + # start_id - The start ID of the range of rows to update. + # end_id - The end ID of the range of rows to update. + def perform(table, copy_from, copy_to, start_id, end_id) + return unless connection.column_exists?(table, copy_to) + + quoted_table = connection.quote_table_name(table) + quoted_copy_from = connection.quote_column_name(copy_from) + quoted_copy_to = connection.quote_column_name(copy_to) + + # We're using raw SQL here since this job may be frequently executed. As + # a result dynamically defining models would lead to many unnecessary + # schema information queries. + connection.execute <<-SQL.strip_heredoc + UPDATE #{quoted_table} + SET #{quoted_copy_to} = #{quoted_copy_from} + WHERE id BETWEEN #{start_id} AND #{end_id} + AND #{quoted_copy_from} IS NOT NULL + AND #{quoted_copy_to} IS NULL + SQL + end + + def connection + ActiveRecord::Base.connection + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb index a1af045a71f..21b626dde56 100644 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -1,44 +1,12 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab module BackgroundMigration class DeleteConflictingRedirectRoutesRange - class Route < ActiveRecord::Base - self.table_name = 'routes' - end - - class RedirectRoute < ActiveRecord::Base - self.table_name = 'redirect_routes' - end - - # start_id - The start ID of the range of events to process - # end_id - The end ID of the range to process. def perform(start_id, end_id) - return unless migrate? - - conflicts = RedirectRoute.where(routes_match_redirects_clause(start_id, end_id)) - num_rows = conflicts.delete_all - - Rails.logger.info("Gitlab::BackgroundMigration::DeleteConflictingRedirectRoutesRange [#{start_id}, #{end_id}] - Deleted #{num_rows} redirect routes that were conflicting with routes.") - end - - def migrate? - Route.table_exists? && RedirectRoute.table_exists? - end - - def routes_match_redirects_clause(start_id, end_id) - <<~ROUTES_MATCH_REDIRECTS - EXISTS ( - SELECT 1 FROM routes - WHERE ( - LOWER(redirect_routes.path) = LOWER(routes.path) - OR LOWER(redirect_routes.path) LIKE LOWER(CONCAT(routes.path, '/%')) - ) - AND routes.id BETWEEN #{start_id} AND #{end_id} - ) - ROUTES_MATCH_REDIRECTS + # No-op. + # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 end end end diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 84ac00f1a5c..7088aa0860a 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -128,8 +128,14 @@ module Gitlab end def process_event(event) - replicate_event(event) - create_push_event_payload(event) if event.push_event? + ActiveRecord::Base.transaction do + replicate_event(event) + create_push_event_payload(event) if event.push_event? + end + rescue ActiveRecord::InvalidForeignKey => e + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + Rails.logger.error("Unable to migrate event #{event.id}: #{e}") end def replicate_event(event) @@ -137,9 +143,6 @@ module Gitlab .with_indifferent_access.except(:title, :data) EventForMigration.create!(new_attributes) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def create_push_event_payload(event) @@ -156,9 +159,6 @@ module Gitlab ref: event.trimmed_ref_name, commit_title: event.commit_title ) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def find_events(start_id, end_id) diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb new file mode 100644 index 00000000000..8a901a9bf39 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/ClassLength +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateMergeRequestMetricsWithEventsData + def perform(min_merge_request_id, max_merge_request_id) + insert_metrics_for_range(min_merge_request_id, max_merge_request_id) + update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) + end + + # Inserts merge_request_metrics records for merge_requests without it for + # a given merge request batch. + def insert_metrics_for_range(min, max) + metrics_not_exists_clause = + <<-SQL.strip_heredoc + NOT EXISTS (SELECT 1 FROM merge_request_metrics + WHERE merge_request_metrics.merge_request_id = merge_requests.id) + SQL + + MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| + select_sql = batch.select(:id, :created_at, :updated_at).to_sql + + execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") + end + end + + def update_metrics_with_events_data(min, max) + if Gitlab::Database.postgresql? + # Uses WITH syntax in order to update merged and closed events with a single UPDATE. + # WITH is not supported by MySQL. + update_events_for_range(min, max) + else + update_merged_events_for_range(min, max) + update_closed_events_for_range(min, max) + end + end + + private + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id and merged_by_id + # based on the latest event records on events table for a given merge request batch. + def update_events_for_range(min, max) + sql = <<-SQL.strip_heredoc + WITH events_for_update AS ( + SELECT DISTINCT ON (target_id, action) target_id, action, author_id, updated_at + FROM events + WHERE target_id BETWEEN #{min} AND #{max} + AND target_type = 'MergeRequest' + AND action IN (#{Event::CLOSED},#{Event::MERGED}) + ORDER BY target_id, action, id DESC + ) + UPDATE merge_request_metrics met + SET latest_closed_at = latest_closed.updated_at, + latest_closed_by_id = latest_closed.author_id, + merged_by_id = latest_merged.author_id + FROM (SELECT * FROM events_for_update WHERE action = #{Event::CLOSED}) AS latest_closed + FULL OUTER JOIN + (SELECT * FROM events_for_update WHERE action = #{Event::MERGED}) AS latest_merged + USING (target_id) + WHERE target_id = merge_request_id; + SQL + + execute(sql) + end + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id based on the latest closed + # records on events table for a given merge request batch. + def update_closed_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::CLOSED)}) closed_events + SET metrics.latest_closed_by_id = closed_events.author_id, + metrics.latest_closed_at = closed_events.updated_at #{where_matches_closed_events}; + SQL + + execute(sql) + end + + # Updates merge_request_metrics merged_by_id based on the latest merged + # records on events table for a given merge request batch. + def update_merged_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::MERGED)}) merged_events + SET metrics.merged_by_id = merged_events.author_id #{where_matches_merged_events}; + SQL + + execute(sql) + end + + def execute(sql) + @connection ||= ActiveRecord::Base.connection + @connection.execute(sql) + end + + def select_events(min, max, action) + select_max_event_id = <<-SQL.strip_heredoc + SELECT max(id) + FROM events + WHERE action = #{action} + AND target_type = 'MergeRequest' + AND target_id BETWEEN #{min} AND #{max} + GROUP BY target_id + SQL + + <<-SQL.strip_heredoc + SELECT author_id, updated_at, target_id + FROM events + WHERE id IN(#{select_max_event_id}) + SQL + end + + def where_matches_closed_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = closed_events.target_id + AND metrics.latest_closed_at IS NULL + AND metrics.latest_closed_by_id IS NULL + SQL + end + + def where_matches_merged_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = merged_events.target_id + AND metrics.merged_by_id IS NULL + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 81e95e5832d..8a8e770940e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -12,7 +12,7 @@ module Gitlab # Ends with /:random_hex/:filename FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} - FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}} + FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -143,7 +143,7 @@ module Gitlab end def absolute_path - File.join(CarrierWave.root, path) + File.join(Gitlab.config.uploads.storage_path, path) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 476c46341ae..a7a1bbe1752 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -7,12 +7,16 @@ module Gitlab class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength # For bulk_queue_background_migration_jobs_by_range include Database::MigrationHelpers + include ::Gitlab::Utils::StrongMemoize FIND_BATCH_SIZE = 500 RELATIVE_UPLOAD_DIR = "uploads".freeze - ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze + ABSOLUTE_UPLOAD_DIR = File.join( + Gitlab.config.uploads.storage_path, + RELATIVE_UPLOAD_DIR + ) FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} + START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/} EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze @@ -80,7 +84,7 @@ module Gitlab paths = [] stdout.each_line("\0") do |line| - paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') + paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '') if paths.size >= batch_size yield(paths) @@ -142,7 +146,9 @@ module Gitlab end def postgresql? - @postgresql ||= Gitlab::Database.postgresql? + strong_memoize(:postgresql) do + Gitlab::Database.postgresql? + end end def can_bulk_insert_and_ignore_duplicates? @@ -150,8 +156,9 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && - Gitlab::Database.version.to_f < 9.5 + strong_memoize(:postgresql_pre_9_5) do + postgresql? && Gitlab::Database.version.to_f < 9.5 + end end def schedule_populate_untracked_uploads_jobs diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 9a0482306b7..778d78185ff 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -23,7 +23,7 @@ module Gitlab @coverage ||= raw_coverage return unless @coverage - @coverage.to_i + @coverage.to_f.round(2) end def metadata diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index fcecb1d9665..afbf9dd17e3 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -25,7 +25,7 @@ module Gitlab end def value_text - @status ? "#{@status}%" : 'unknown' + @status ? ("%.2f%%" % @status) : 'unknown' end def key_width @@ -33,7 +33,7 @@ module Gitlab end def value_width - @status ? 36 : 58 + @status ? 54 : 58 end def value_color diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 298409d8b5a..884a3de8f62 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -14,7 +14,7 @@ module Gitlab repos_to_import.each do |repo_path| bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) - if bare_repo.hashed? || bare_repo.wiki? + unless bare_repo.processable? log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) next @@ -55,12 +55,16 @@ module Gitlab name: project_name, path: project_name, skip_disk_validation: true, - import_type: 'gitlab_project', + skip_wiki: bare_repo.wiki_exists?, + import_type: 'bare_repository', namespace_id: group&.id).execute if project.persisted? && mv_repo(project) log " * Created #{project.name} (#{project_full_path})".color(:green) + project.write_repository_config + project.repository.create_hooks + ProjectCacheWorker.perform_async(project.id) else log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fa7891c8dcc..fe267248275 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -1,44 +1,69 @@ +# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953 +# module Gitlab module BareRepositoryImport class Repository + include ::Gitlab::Utils::StrongMemoize + attr_reader :group_path, :project_name, :repo_path def initialize(root_path, repo_path) @root_path = root_path @repo_path = repo_path - @root_path << '/' unless root_path.ends_with?('/') + full_path = + if hashed? && !wiki? + repository.config.get('gitlab.fullpath') + else + repo_relative_path + end + # Split path into 'all/the/namespaces' and 'project_name' - @group_path, _, @project_name = repo_relative_path.rpartition('/') + @group_path, _, @project_name = full_path.to_s.rpartition('/') end def wiki_exists? File.exist?(wiki_path) end - def wiki? - @wiki ||= repo_path.end_with?('.wiki.git') - end - def wiki_path @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') end - def hashed? - @hashed ||= group_path.start_with?('@hashed') - end - def project_full_path @project_full_path ||= "#{group_path}/#{project_name}" end + def processable? + return false if wiki? + return false if hashed? && (group_path.blank? || project_name.blank?) + + true + end + private + def wiki? + strong_memoize(:wiki) do + repo_path.end_with?('.wiki.git') + end + end + + def hashed? + strong_memoize(:hashed) do + repo_relative_path.include?('@hashed') + end + end + def repo_relative_path # Remove root path and `.git` at the end repo_path[@root_path.size...-4] end + + def repository + @repository ||= Rugged::Repository.new(repo_path) + end end end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index ef92fc5a0a0..945d70e7a24 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -16,7 +16,7 @@ module Gitlab lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze - attr_reader :user_access, :project, :skip_authorization, :protocol + attr_reader :user_access, :project, :skip_authorization, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name def initialize( change, user_access:, project:, skip_authorization: false, @@ -51,9 +51,9 @@ module Gitlab end def branch_checks - return unless @branch_name + return unless branch_name - if deletion? && @branch_name == project.default_branch + if deletion? && branch_name == project.default_branch raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] end @@ -61,7 +61,7 @@ module Gitlab end def protected_branch_checks - return unless ProtectedBranch.protected?(project, @branch_name) + return unless ProtectedBranch.protected?(project, branch_name) if forced_push? raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] @@ -75,29 +75,29 @@ module Gitlab end def protected_branch_deletion_checks - unless user_access.can_delete_branch?(@branch_name) + unless user_access.can_delete_branch?(branch_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] end - unless protocol == 'web' + unless updated_from_web? raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] end end def protected_branch_push_checks if matching_merge_request? - unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) + unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] end else - unless user_access.can_push_to_branch?(@branch_name) + unless user_access.can_push_to_branch?(branch_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch] end end end def tag_checks - return unless @tag_name + return unless tag_name if tag_exists? && user_access.cannot_do_action?(:admin_project) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] @@ -107,40 +107,44 @@ module Gitlab end def protected_tag_checks - return unless ProtectedTag.protected?(project, @tag_name) + return unless ProtectedTag.protected?(project, tag_name) raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? - unless user_access.can_create_tag?(@tag_name) + unless user_access.can_create_tag?(tag_name) raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] end end private + def updated_from_web? + protocol == 'web' + end + def tag_exists? - project.repository.tag_exists?(@tag_name) + project.repository.tag_exists?(tag_name) end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) end def update? - !Gitlab::Git.blank_ref?(@oldrev) && !deletion? + !Gitlab::Git.blank_ref?(oldrev) && !deletion? end def deletion? - Gitlab::Git.blank_ref?(@newrev) + Gitlab::Git.blank_ref?(newrev) end def matching_merge_request? - Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match? + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end def lfs_objects_exist_check - lfs_check = Checks::LfsIntegrity.new(project, @newrev) + lfs_check = Checks::LfsIntegrity.new(project, newrev) if lfs_check.objects_missing? raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3a1c0a3455e..dfb2f4d4054 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -21,6 +21,10 @@ module Gitlab end def add_redirect_message + # Don't bother with sending a redirect message for anonymous clones + # because they never see it via the `/internal/post_receive` endpoint + return unless user.present? && project.present? + Gitlab::Redis::SharedState.with do |redis| key = self.class.redirect_message_key(user.id, project.id) redis.setex(key, 5.minutes, redirect_message) diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 72b75791bbb..35eadf6fa93 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -148,6 +148,7 @@ module Gitlab stream.seek(@offset) append = @offset > 0 end + start_offset = @offset open_new_tag @@ -155,6 +156,7 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? + if s.scan(Gitlab::Regex.build_trace_section_regex) handle_section(s) elsif s.scan(/\e([@-_])(.*?)([@-~])/) @@ -168,6 +170,7 @@ module Gitlab else @out << s.scan(/./m) end + @offset += s.matched_size end end @@ -234,10 +237,12 @@ module Gitlab # Most terminals show bold colored text in the light color variant # Let's mimic that here if @style_mask & STYLE_SWITCHES[:bold] != 0 - fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + fg_color.sub!(/fg-([a-z]{2,}+)/, 'fg-l-\1') end + css_classes << fg_color end + css_classes << @bg_color unless @bg_color.nil? STYLE_SWITCHES.each do |css_class, flag| diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 5b2f09e03ea..428c0505808 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -97,7 +97,7 @@ module Gitlab end def total_size - descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}} + descendant_pattern = /^#{Regexp.escape(@path.to_s)}/ entries.sum do |path, entry| (entry[:size] if path =~ descendant_pattern).to_i end diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index 1fba0b2db0b..26505c91be3 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -93,9 +93,7 @@ module Gitlab private - def entries - @entries - end + attr_reader :entries end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index eb606b57667..55658900628 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -64,10 +64,24 @@ module Gitlab include LegacyValidationHelpers def validate_each(record, attribute, value) - unless validate_string(value) + if validate_string(value) + validate_path(record, attribute, value) + else record.errors.add(attribute, 'should be a string or symbol') end end + + private + + def validate_path(record, attribute, value) + path = CGI.unescape(value.to_s) + + if path.include?('/') + record.errors.add(attribute, 'cannot contain the "/" character') + elsif path == '.' || path == '..' + record.errors.add(attribute, 'cannot be "." or ".."') + end + end end class RegexpValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 9a72de87bab..32cbb7ca6af 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -3,6 +3,8 @@ module Gitlab module Pipeline module Chain class Skip < Chain::Base + include ::Gitlab::Utils::StrongMemoize + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i def perform! @@ -24,7 +26,9 @@ module Gitlab def commit_message_skips_ci? return false unless @pipeline.git_commit_message - @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + strong_memoize(:commit_message_skips_ci) do + !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + end end end end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index bc97aa63b02..f33c87f554d 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -2,6 +2,8 @@ module Gitlab module Ci module Stage class Seed + include ::Gitlab::Utils::StrongMemoize + attr_reader :pipeline delegate :project, to: :pipeline @@ -50,7 +52,9 @@ module Gitlab private def protected_ref? - @protected_ref ||= project.protected_for?(pipeline.ref) + strong_memoize(:protected_ref) do + project.protected_for?(pipeline.ref) + end end end end diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb index 45fd0d4aa07..6c9125647ad 100644 --- a/lib/gitlab/ci/status/build/action.rb +++ b/lib/gitlab/ci/status/build/action.rb @@ -2,6 +2,9 @@ module Gitlab module Ci module Status module Build + ## + # Extended status for playable manual actions. + # class Action < Status::Extended def label if has_action? @@ -12,7 +15,7 @@ module Gitlab end def self.matches?(build, user) - build.action? + build.playable? end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 76aee5a3deb..0a3ae2c3760 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -13,12 +13,13 @@ module Gitlab end def resolve(user, commit_message, files) + msg = commit_message || default_commit_message + resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg) args = { source_branch: merge_request.source_branch, - target_branch: merge_request.target_branch, - commit_message: commit_message || default_commit_message + target_branch: merge_request.target_branch } - resolver.resolve_conflicts(@source_repo, user, files, args) + resolver.resolve_conflicts(@source_repo, resolution, args) ensure @merge_request.clear_memoized_shas end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 91fd9cc7631..b7c596a973d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,73 +1,79 @@ module Gitlab module CurrentSettings - extend self + class << self + def current_application_settings + if RequestStore.active? + RequestStore.fetch(:current_application_settings) { ensure_application_settings! } + else + ensure_application_settings! + end + end - def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! + def fake_application_settings(defaults = ::ApplicationSetting.defaults) + Gitlab::FakeApplicationSettings.new(defaults) end - end - delegate :sidekiq_throttling_enabled?, to: :current_application_settings + def method_missing(name, *args, &block) + current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end - def fake_application_settings(defaults = ::ApplicationSetting.defaults) - FakeApplicationSettings.new(defaults) - end + def respond_to_missing?(name, include_private = false) + current_application_settings.respond_to?(name, include_private) || super + end - private + private - def ensure_application_settings! - return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + def ensure_application_settings! + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - cached_application_settings || uncached_application_settings - end + cached_application_settings || uncached_application_settings + end - def cached_application_settings - begin - ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL - # In case Redis isn't running or the Redis UNIX socket file is not available + def cached_application_settings + begin + ::ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL + # In case Redis isn't running or the Redis UNIX socket file is not available + end end - end - def uncached_application_settings - return fake_application_settings unless connect_to_db? + def uncached_application_settings + return fake_application_settings unless connect_to_db? - db_settings = ::ApplicationSetting.current + db_settings = ::ApplicationSetting.current - # If there are pending migrations, it's possible there are columns that - # need to be added to the application settings. To prevent Rake tasks - # and other callers from failing, use any loaded settings and return - # defaults for missing columns. - if ActiveRecord::Migrator.needs_migration? - defaults = ::ApplicationSetting.defaults - defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? - return fake_application_settings(defaults) - end + # If there are pending migrations, it's possible there are columns that + # need to be added to the application settings. To prevent Rake tasks + # and other callers from failing, use any loaded settings and return + # defaults for missing columns. + if ActiveRecord::Migrator.needs_migration? + defaults = ::ApplicationSetting.defaults + defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? + return fake_application_settings(defaults) + end - return db_settings if db_settings.present? + return db_settings if db_settings.present? - ::ApplicationSetting.create_from_defaults || in_memory_application_settings - end + ::ApplicationSetting.create_from_defaults || in_memory_application_settings + end - def in_memory_application_settings - @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables - rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct - fake_application_settings - end + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + fake_application_settings + end - def connect_to_db? - # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised - active_db_connection = ActiveRecord::Base.connection.active? rescue false + def connect_to_db? + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false - active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - rescue ActiveRecord::NoDatabaseError - false + active_db_connection && + ActiveRecord::Base.connection.table_exists?('application_settings') + rescue ActiveRecord::NoDatabaseError + false + end end end end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index dcbdf9a64b0..8b3bc3e440d 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -15,7 +15,6 @@ module Gitlab query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) .where(issue_table[:project_id].eq(@project.id)) # rubocop:disable Gitlab/ModuleWithInstanceVariables - .where(issue_table[:deleted_at].eq(nil)) .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables # Load merge_requests diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 9f76967fc77..d32837f5793 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -12,30 +12,40 @@ module Gitlab # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - priv = - if Database.postgresql? - where(privilege_type: 'TRIGGER', table_name: table) - .where('grantee = user') - else - queries = [ - Grant.select(1) - .from('information_schema.user_privileges') - .where("PRIVILEGE_TYPE = 'SUPER'") - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), + if Database.postgresql? + # We _must not_ use quote_table_name as this will produce double + # quotes on PostgreSQL and for "has_table_privilege" we need single + # quotes. + quoted_table = connection.quote(table) - Grant.select(1) - .from('information_schema.schema_privileges') - .where("PRIVILEGE_TYPE = 'TRIGGER'") - .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") - ] + begin + from(nil) + .pluck("has_table_privilege(#{quoted_table}, 'TRIGGER')") + .first + rescue ActiveRecord::StatementInvalid + # This error is raised when using a non-existing table name. In this + # case we just want to return false as a user technically can't + # create triggers for such a table. + false + end + else + queries = [ + Grant.select(1) + .from('information_schema.user_privileges') + .where("PRIVILEGE_TYPE = 'SUPER'") + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), - union = SQL::Union.new(queries).to_sql + Grant.select(1) + .from('information_schema.schema_privileges') + .where("PRIVILEGE_TYPE = 'TRIGGER'") + .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") + ] - Grant.from("(#{union}) privs") - end + union = SQL::Union.new(queries).to_sql - priv.any? + Grant.from("(#{union}) privs").any? + end end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3f65bc912de..dbe6259fce7 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -385,10 +385,27 @@ module Gitlab # necessary since we copy over old values further down. change_column_default(table, new, old_col.default) if old_col.default - trigger_name = rename_trigger_name(table, old, new) + install_rename_triggers(table, old, new) + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + change_column_null(table, new, false) unless old_col.null + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + + # Installs triggers in a table that keep a new column in sync with an old + # one. + # + # table - The name of the table to install the trigger in. + # old_column - The name of the old column. + # new_column - The name of the new column. + def install_rename_triggers(table, old_column, new_column) + trigger_name = rename_trigger_name(table, old_column, new_column) quoted_table = quote_table_name(table) - quoted_old = quote_column_name(old) - quoted_new = quote_column_name(new) + quoted_old = quote_column_name(old_column) + quoted_new = quote_column_name(new_column) if Database.postgresql? install_rename_triggers_for_postgresql(trigger_name, quoted_table, @@ -397,13 +414,6 @@ module Gitlab install_rename_triggers_for_mysql(trigger_name, quoted_table, quoted_old, quoted_new) end - - update_column_in_batches(table, new, Arel::Table.new(table)[old]) - - change_column_null(table, new, false) unless old_col.null - - copy_indexes(table, old, new) - copy_foreign_keys(table, old, new) end # Changes the type of a column concurrently. @@ -455,6 +465,99 @@ module Gitlab remove_column(table, old) end + # Changes the column type of a table using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `change_column_type_concurrently` since it can complete its work in a + # much shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # class Issue < ActiveRecord::Base + # self.table_name = 'issues' + # + # include EachBatch + # + # def self.to_migrate + # where('closed_at IS NOT NULL') + # end + # end + # + # change_column_type_using_background_migration( + # Issue.to_migrate, + # :closed_at, + # :datetime_with_timezone + # ) + # + # Reverting a migration like this is done exactly the same way, just with + # a different type to migrate to (e.g. `:datetime` in the above example). + # + # relation - An ActiveRecord relation to use for scheduling jobs and + # figuring out what table we're modifying. This relation _must_ + # have the EachBatch module included. + # + # column - The name of the column for which the type will be changed. + # + # new_type - The new type of the column. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def change_column_type_using_background_migration( + relation, + column, + new_type, + batch_size: 10_000, + interval: 10.minutes + ) + + unless relation.model < EachBatch + raise TypeError, 'The relation must include the EachBatch module' + end + + temp_column = "#{column}_for_type_change" + table = relation.table_name + max_index = 0 + + add_column(table, temp_column, new_type) + install_rename_triggers(table, column, temp_column) + + # Schedule the jobs that will copy the data from the old column to the + # new one. Rows with NULL values in our source column are skipped since + # the target column is already NULL at this point. + relation.where.not(column => nil).each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, column, temp_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentTypeChange', + [table, column, temp_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange') + end + end + + def perform_background_migration_inline? + Rails.env.test? || Rails.env.development? + end + # Performs a concurrent column rename when using PostgreSQL. def install_rename_triggers_for_postgresql(trigger, table, old, new) execute <<-EOF.strip_heredoc @@ -741,6 +844,12 @@ into similar problems in the future (e.g. when new tables are created). def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + # To not overload the worker too much we enforce a minimum interval both + # when scheduling and performing jobs. + if delay_interval < BackgroundMigrationWorker::MIN_INTERVAL + delay_interval = BackgroundMigrationWorker::MIN_INTERVAL + end + model_class.each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck('MIN(id), MAX(id)').first diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index d32616862f0..979225dd216 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -26,6 +26,7 @@ module Gitlab move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") end + move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) move_pages(old_full_path, new_full_path) end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index 0245bf4077a..cfd4ec15125 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -11,7 +11,7 @@ module Gitlab end def package_url(name) - "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z} + "https://packagist.org/packages/#{name}" if name =~ /\A#{REPO_REGEX}\z/ end end end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index d034ea67387..bfea836bcb2 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -15,7 +15,7 @@ module Gitlab link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) # Link `source "https://rubygems.org"` to https://rubygems.org link_method_call('source', URL_REGEX, &:itself) diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index a52c7a02439..924e55e9820 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -12,7 +12,7 @@ module Gitlab def link_dependencies link_method_call('homepage', URL_REGEX, &:itself) - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) link_method_call('license', &method(:license_url)) link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d0cfe2386ca..34b070dd375 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -61,7 +61,9 @@ module Gitlab end def line_for_position(pos) - diff_lines.find { |line| position(line) == pos } + return nil unless pos.position_type == 'text' + + diff_lines.find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } end def position_for_line_code(code) @@ -114,8 +116,10 @@ module Gitlab new_content_sha || old_content_sha end + # Use #itself to check the value wrapped by a BatchLoader instance, rather + # than if the BatchLoader instance itself is falsey. def blob - new_blob || old_blob + new_blob&.itself || old_blob&.itself end attr_writer :highlighted_diff_lines @@ -171,7 +175,7 @@ module Gitlab end def binary? - has_binary_notice? || old_blob&.binary? || new_blob&.binary? + has_binary_notice? || try_blobs(:binary?) end def text? @@ -179,15 +183,15 @@ module Gitlab end def external_storage_error? - old_blob&.external_storage_error? || new_blob&.external_storage_error? + try_blobs(:external_storage_error?) end def stored_externally? - old_blob&.stored_externally? || new_blob&.stored_externally? + try_blobs(:stored_externally?) end def external_storage - old_blob&.external_storage || new_blob&.external_storage + try_blobs(:external_storage) end def content_changed? @@ -202,15 +206,15 @@ module Gitlab end def size - [old_blob&.size, new_blob&.size].compact.sum + valid_blobs.map(&:size).sum end def raw_size - [old_blob&.raw_size, new_blob&.raw_size].compact.sum + valid_blobs.map(&:raw_size).sum end def raw_binary? - old_blob&.raw_binary? || new_blob&.raw_binary? + try_blobs(:raw_binary?) end def raw_text? @@ -233,6 +237,19 @@ module Gitlab private + # The blob instances are instances of BatchLoader, which means calling + # &. directly on them won't work. Object#try also won't work, because Blob + # doesn't inherit from Object, but from BasicObject (via SimpleDelegator). + def try_blobs(meth) + old_blob&.itself&.public_send(meth) || new_blob&.itself&.public_send(meth) + end + + # We can't use #compact for the same reason we can't use &., but calling + # #nil? explicitly does work because it is proxied to the blob itself. + def valid_blobs + [old_blob, new_blob].reject(&:nil?) + end + def text_position_properties(line) { old_line: line.old_line, new_line: line.new_line } end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index b669ee5b799..0f897e6316c 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -14,6 +14,7 @@ module Gitlab else @diff_lines = diff_lines end + @raw_lines = @diff_lines.map(&:text) end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 37face8e7d0..0fb71976883 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -5,7 +5,7 @@ module Gitlab DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze + IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== @@ -156,12 +156,14 @@ module Gitlab %W[git apply --3way #{patch_path}] ) do |output, status| puts output + unless status.zero? @failed_files = output.lines.reduce([]) do |memo, line| if line.start_with?('error: patch failed:') file = line.sub(/\Aerror: patch failed: /, '') memo << file unless file =~ IGNORED_FILES_REGEX end + memo end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e2f7c1d0257..3436306e122 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -10,6 +10,7 @@ module Gitlab def initialize(mail, mail_key) super(mail, mail_key) + if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) @project_path, @incoming_email_token = m.captures end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 558df87f36d..01c28d051ee 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -43,7 +43,7 @@ module Gitlab return "" unless decoded # Certain trigger phrases that means we didn't parse correctly - if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + if decoded =~ %r{(Content\-Type\:|multipart/alternative|text/plain)} return "" end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 582028493e9..c0edcabc6fd 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,14 +14,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) - return nil unless message.respond_to?(:force_encoding) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? - - if message.respond_to?(:frozen?) && message.frozen? - message = message.dup - end - - message.force_encoding("UTF-8") + message = force_encode_utf8(message) return message if message.valid_encoding? # return message if message type is binary @@ -35,6 +28,8 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) + rescue ArgumentError + return nil rescue encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" @@ -54,8 +49,8 @@ module Gitlab end def encode_utf8(message) - return nil unless message.is_a?(String) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + message = force_encode_utf8(message) + return message if message.valid_encoding? detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] @@ -69,10 +64,31 @@ module Gitlab else clean(message) end + rescue ArgumentError + return nil + end + + def encode_binary(s) + return "" if s.nil? + + s.dup.force_encoding(Encoding::ASCII_8BIT) + end + + def binary_stringio(s) + StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private + def force_encode_utf8(message) + raise ArgumentError unless message.respond_to?(:force_encoding) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + + message = message.dup if message.respond_to?(:frozen?) && message.frozen? + + message.force_encoding("UTF-8") + end + def clean(message) message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") .encode("UTF-8") diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 3f7b42456af..dbb8f317afe 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -71,5 +71,16 @@ module Gitlab redis.exists(@redis_shared_state_key) end end + + # Returns the TTL of the Redis key. + # + # This method will return `nil` if no TTL could be obtained. + def ttl + Gitlab::Redis::SharedState.with do |redis| + ttl = redis.ttl(@redis_shared_state_key) + + ttl if ttl.positive? + end + end end end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 0e9ef4f897c..cc2638172ec 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -6,14 +6,14 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: /\Areadme[^\/]*\z/i, - changelog: /\A(changelog|history|changes|news)[^\/]*\z/i, - license: /\A(licen[sc]e|copying)(\.[^\/]+)?\z/i, - contributing: /\Acontributing[^\/]*\z/i, + readme: %r{\Areadme[^/]*\z}i, + changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, + license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i, + contributing: %r{\Acontributing[^/]*\z}i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, - issue_template: /\A\.gitlab\/issue_templates\/[^\/]+\.md\z/, - merge_request_template: /\A\.gitlab\/merge_request_templates\/[^\/]+\.md\z/, + issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z}, + merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z}, # Configuration files gitignore: '.gitignore', @@ -22,17 +22,17 @@ module Gitlab route_map: '.gitlab/route-map.yml', # Dependency files - cartfile: /\ACartfile[^\/]*\z/, + cartfile: %r{\ACartfile[^/]*\z}, composer_json: 'composer.json', gemfile: /\A(Gemfile|gems\.rb)\z/, gemfile_lock: 'Gemfile.lock', - gemspec: /\A[^\/]*\.gemspec\z/, + gemspec: %r{\A[^/]*\.gemspec\z}, godeps_json: 'Godeps.json', package_json: 'package.json', podfile: 'Podfile', - podspec_json: /\A[^\/]*\.podspec\.json\z/, - podspec: /\A[^\/]*\.podspec\z/, - requirements_txt: /\A[^\/]*requirements\.txt\z/, + podspec_json: %r{\A[^/]*\.podspec\.json\z}, + podspec: %r{\A[^/]*\.podspec\z}, + requirements_txt: %r{\A[^/]*requirements\.txt\z}, yarn_lock: 'yarn.lock' }.freeze diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 5e426b13ade..8953bc8c148 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -112,6 +112,7 @@ module Gitlab [bug['sCategory'], bug['sPriority']].each do |label| unless label.blank? labels << label + unless @known_labels.include?(label) create_label(label) @known_labels << label @@ -265,6 +266,7 @@ module Gitlab if content.blank? content = '*(No description has been entered for this issue)*' end + body << content body.join("\n\n") @@ -278,6 +280,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 8fab5489616..1b74f735679 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -27,7 +27,7 @@ module Gitlab with_link_in_tmp_dir(file.file) do |open_tmp_file| new_uploader.store!(open_tmp_file) end - new_uploader.to_markdown + new_uploader.markdown_link end end @@ -46,7 +46,7 @@ module Gitlab private def find_file(project, secret, file) - uploader = FileUploader.new(project, secret) + uploader = FileUploader.new(project, secret: secret) uploader.retrieve_from_store!(file) uploader.file end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f7c35cafaa..d4e893b881c 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -6,12 +6,13 @@ module Gitlab CommandError = Class.new(StandardError) CommitError = Class.new(StandardError) + OSError = Class.new(StandardError) class << self include Gitlab::EncodingHelper def ref_name(ref) - encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') end def branch_name(ref) diff --git a/lib/gitlab/git/attributes_at_ref_parser.rb b/lib/gitlab/git/attributes_at_ref_parser.rb new file mode 100644 index 00000000000..26b5bd520d5 --- /dev/null +++ b/lib/gitlab/git/attributes_at_ref_parser.rb @@ -0,0 +1,14 @@ +module Gitlab + module Git + # Parses root .gitattributes file at a given ref + class AttributesAtRefParser + delegate :attributes, to: :@parser + + def initialize(repository, ref) + blob = repository.blob_at(ref, '.gitattributes') + + @parser = AttributesParser.new(blob&.data) + end + end + end +end diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes_parser.rb index 2d20cd473a7..d8aeabb6cba 100644 --- a/lib/gitlab/git/attributes.rb +++ b/lib/gitlab/git/attributes_parser.rb @@ -1,42 +1,26 @@ -# Gitaly note: JV: not sure what to make of this class. Why does it use -# the full disk path of the repository to look up attributes This is -# problematic in Gitaly, because Gitaly hides the full disk path to the -# repository from gitlab-ce. - module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for # file patterns. - # - # Unlike Rugged this parser only needs a single IO call (a call to `open`), - # vastly reducing the time spent in extracting attributes. - # - # This class _only_ supports parsing the attributes file located at - # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files - # (`.gitattributes` is copied to this particular path). - # - # Basic usage: - # - # attributes = Gitlab::Git::Attributes.new(some_repo.path) - # - # attributes.attributes('README.md') # => { "eol" => "lf } - class Attributes - # path - The path to the Git repository. - def initialize(path) - @path = File.expand_path(path) - @patterns = nil + class AttributesParser + def initialize(attributes_data) + @data = attributes_data || "" + + if @data.is_a?(File) + @patterns = parse_file + end end # Returns all the Git attributes for the given path. # - # path - A path to a file for which to get the attributes. + # file_path - A path to a file for which to get the attributes. # # Returns a Hash. - def attributes(path) - full_path = File.join(@path, path) + def attributes(file_path) + absolute_path = File.join('/', file_path) patterns.each do |pattern, attrs| - return attrs if File.fnmatch?(pattern, full_path) + return attrs if File.fnmatch?(pattern, absolute_path) end {} @@ -98,16 +82,10 @@ module Gitlab # Iterates over every line in the attributes file. def each_line - full_path = File.join(@path, 'info/attributes') + @data.each_line do |line| + break unless line.valid_encoding? - return unless File.exist?(full_path) - - File.open(full_path, 'r') do |handle| - handle.each_line do |line| - break unless line.valid_encoding? - - yield line.strip - end + yield line.strip end end @@ -125,7 +103,8 @@ module Gitlab parsed = attrs ? parse_attributes(attrs) : {} - pairs << [File.join(@path, pattern), parsed] + absolute_pattern = File.join('/', pattern) + pairs << [absolute_pattern, parsed] end # Newer entries take precedence over older entries. diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 31effdba292..6d6ed065f79 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -42,9 +42,7 @@ module Gitlab end def load_blame_by_shelling_out - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) - # Read in binary mode to ensure ASCII-8BIT - IO.popen(cmd, 'rb') {|io| io.read } + @repo.shell_blame(@sha, @path) end def process_raw_blame(output) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 228d97a87ab..4828301dbb9 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -34,7 +34,7 @@ module Gitlab def raw(repository, sha) Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) end @@ -50,10 +50,19 @@ module Gitlab # to the caller to limit the number of blobs and blob_size_limit. # # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 - def batch(repository, blob_references, blob_size_limit: nil) - blob_size_limit ||= MAX_DATA_DISPLAY_SIZE - blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| + if is_enabled + Gitlab::GitalyClient.allow_n_plus_1_calls do + blob_references.map do |sha, path| + find_by_gitaly(repository, sha, path, limit: blob_size_limit) + end + end + else + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end end end @@ -61,11 +70,17 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - blob_ids.lazy - .select { |sha| possible_lfs_blob?(repository, sha) } - .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } - .select(&:lfs_pointer?) - .force + repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled| + if is_enabled + repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) + else + blob_ids.lazy + .select { |sha| possible_lfs_blob?(repository, sha) } + .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } + .select(&:lfs_pointer?) + .force + end + end end def binary?(data) @@ -92,7 +107,7 @@ module Gitlab def find_entry_by_path(repository, root_id, path) root_tree = repository.lookup(root_id) # Strip leading slashes - path[/^\/*/] = '' + path[%r{^/*}] = '' path_arr = path.split('/') entry = root_tree.find do |entry| @@ -122,13 +137,25 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path) - path = path.sub(/\A\/*/, '') + def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + + path = path.sub(%r{\A/*}, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) return unless entry + entry.data = "" if limit == 0 + case entry.type when :COMMIT new( @@ -154,8 +181,10 @@ module Gitlab end def find_by_rugged(repository, sha, path, limit:) - commit = repository.lookup(sha) - root_tree = commit.tree + return unless path + + rugged_commit = repository.lookup(sha) + root_tree = rugged_commit.tree blob_entry = find_entry_by_path(repository, root_tree.oid, path) @@ -235,7 +264,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| @data = begin if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data + repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else repository.lookup(id).content end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 3487e099381..ae7e88f0503 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. - module Gitlab module Git class Branch < Ref diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 145721dea76..768617e2cae 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -15,8 +15,6 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator - delegate :tree, to: :rugged_commit - def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -241,6 +239,24 @@ module Gitlab end end end + + def extract_signature(repository, commit_id) + repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| + if is_enabled + repository.gitaly_commit_client.extract_signature(commit_id) + else + rugged_extract_signature(repository, commit_id) + end + end + end + + def rugged_extract_signature(repository, commit_id) + begin + Rugged::Commit.extract_signature(repository.rugged, commit_id) + rescue Rugged::OdbError + nil + end + end end def initialize(repository, raw_commit, head = nil) @@ -438,6 +454,16 @@ module Gitlab parent_ids.size > 1 end + def tree_entry(path) + @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| + if is_migrated + gitaly_tree_entry(path) + else + rugged_tree_entry(path) + end + end + end + def to_gitaly_commit return raw_commit if raw_commit.is_a?(Gitaly::GitCommit) @@ -498,6 +524,28 @@ module Gitlab SERIALIZE_KEYS end + def gitaly_tree_entry(path) + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + # To be compatible with the rugged format + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry + end + + # Is this the same as Blob.find_entry_by_path ? + def rugged_tree_entry(path) + rugged_commit.tree.path(path) + rescue Rugged::TreeError + nil + end + def gitaly_commit_author_from_rugged(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 6bf49a0af18..8463b1eb794 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -34,13 +34,8 @@ module Gitlab def rugged_stats(commit) diff = commit.rugged_diff_from_parent - - diff.each_patch do |p| - # TODO: Use the new Rugged convenience methods when they're released - @additions += p.stat[0] - @deletions += p.stat[1] - @total += p.changes - end + _files_changed, @additions, @deletions = diff.stat + @total = @additions + @deletions end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index b2a625e08fa..2a9cf10a068 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,7 +2,9 @@ module Gitlab module Git module Conflict class File - attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid + attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid + + attr_accessor :content def initialize(repository, commit_oid, conflict, content) @repository = repository diff --git a/lib/gitlab/git/conflict/resolution.rb b/lib/gitlab/git/conflict/resolution.rb new file mode 100644 index 00000000000..ab9be683e15 --- /dev/null +++ b/lib/gitlab/git/conflict/resolution.rb @@ -0,0 +1,15 @@ +module Gitlab + module Git + module Conflict + class Resolution + attr_reader :user, :files, :commit_message + + def initialize(user, files, commit_message) + @user = user + @files = files + @commit_message = commit_message + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 03e5c0fcd6f..07b7e811a34 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -13,37 +13,27 @@ module Gitlab def conflicts @conflicts ||= begin - target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - conflict_files(@target_repository, target_index) + @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| + if is_enabled + gitaly_conflicts_client(@target_repository).list_conflict_files.to_a + else + rugged_list_conflict_files + end + end end + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) + rescue Rugged::OdbError, GRPC::BadStatus => e + raise Gitlab::Git::CommandError.new(e) end - def resolve_conflicts(source_repository, user, files, source_branch:, target_branch:, commit_message:) - source_repository.with_repo_branch_commit(@target_repository, target_branch) do - index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - conflicts = conflict_files(source_repository, index) - - files.each do |file_params| - conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) - - write_resolved_file_to_index(source_repository, index, conflict_file, file_params) - end - - unless index.conflicts.empty? - missing_files = index.conflicts.map { |file| file[:ours][:path] } - - raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) + source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled| + if is_enabled + gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) + else + rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) end - - commit_params = { - message: commit_message, - parents: [@our_commit_oid, @their_commit_oid] - } - - source_repository.commit_index(user, source_branch, index, commit_params) end end @@ -68,6 +58,10 @@ module Gitlab end end + def gitaly_conflicts_client(repository) + repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid) + end + def write_resolved_file_to_index(repository, index, file, params) if params[:sections] resolved_lines = file.resolve_lines(params[:sections]) @@ -84,6 +78,40 @@ module Gitlab index.add(path: our_path, oid: oid, mode: file.our_mode) index.conflict_remove(our_path) end + + def rugged_list_conflict_files + target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + conflict_files(@target_repository, target_index) + end + + def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) + source_repository.with_repo_branch_commit(@target_repository, target_branch) do + index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + conflicts = conflict_files(source_repository, index) + + resolution.files.each do |file_params| + conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) + + write_resolved_file_to_index(source_repository, index, conflict_file, file_params) + end + + unless index.conflicts.empty? + missing_files = index.conflicts.map { |file| file[:ours][:path] } + + raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + end + + commit_params = { + message: resolution.commit_message, + parents: [@our_commit_oid, @their_commit_oid] + } + + source_repository.commit_index(resolution.user, source_branch, index, commit_params) + end + end end end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index ca94b4baa59..a203587aec1 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -44,7 +44,7 @@ module Gitlab # branch1...branch2) From the git documentation: # "git diff A...B" is equivalent to "git diff # $(git-merge-base A B) B" - repo.merge_base_commit(head, base) + repo.merge_base(head, base) end options ||= {} diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index d948d7895ed..e5a747cb987 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -2,6 +2,9 @@ module Gitlab module Git class GitlabProjects include Gitlab::Git::Popen + include Gitlab::Utils::StrongMemoize + + ShardNameNotFoundError = Class.new(StandardError) # Absolute path to directory where repositories are stored. # Example: /home/git/repositories @@ -38,81 +41,26 @@ module Gitlab io.read end - def rm_project - logger.info "Removing repository <#{repository_absolute_path}>." - FileUtils.rm_rf(repository_absolute_path) - end - - # Move repository from one directory to another - # - # Example: gitlab/gitlab-ci.git -> randx/six.git - # - # Won't work if target namespace directory does not exist - # - def mv_project(new_path) - new_absolute_path = File.join(shard_path, new_path) - - # verify that the source repo exists - unless File.exist?(repository_absolute_path) - logger.error "mv-project failed: source path <#{repository_absolute_path}> does not exist." - return false - end - - # ...and that the target repo does not exist - if File.exist?(new_absolute_path) - logger.error "mv-project failed: destination path <#{new_absolute_path}> already exists." - return false - end - - logger.info "Moving repository from <#{repository_absolute_path}> to <#{new_absolute_path}>." - FileUtils.mv(repository_absolute_path, new_absolute_path) - end - # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false + Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled| + if is_enabled + gitaly_import_repository(source) + else + git_import_repository(source, timeout) + end end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true end def fork_repository(new_shard_path, new_repository_relative_path) - from_path = repository_absolute_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false + Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| + if is_enabled + gitaly_fork_repository(new_shard_path, new_repository_relative_path) + else + git_fork_repository(new_shard_path, new_repository_relative_path) + end end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil) @@ -253,6 +201,84 @@ module Gitlab known_hosts_file&.close! script&.close! end + + private + + def shard_name + strong_memoize(:shard_name) do + shard_name_from_shard_path(shard_path) + end + end + + def shard_name_from_shard_path(shard_path) + Gitlab.config.repositories.storages.find { |_, info| info['path'] == shard_path }&.first || + raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") + end + + def git_import_repository(source, timeout) + # Skip import if repo already exists + return false if File.exist?(repository_absolute_path) + + masked_source = mask_password_in_url(source) + + logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." + cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) + + success = run_with_timeout(cmd, timeout, nil) + + unless success + logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") + FileUtils.rm_rf(repository_absolute_path) + return false + end + + Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) + + # The project was imported successfully. + # Remove the origin URL since it may contain password. + remove_origin_in_repo + + true + end + + def gitaly_import_repository(source) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output << e.message + false + end + + def git_fork_repository(new_shard_path, new_repository_relative_path) + from_path = repository_absolute_path + to_path = File.join(new_shard_path, new_repository_relative_path) + + # The repository cannot already exist + if File.exist?(to_path) + logger.error "fork-repository failed: destination repository <#{to_path}> already exists." + return false + end + + # Ensure the namepsace / hashed storage directory exists + FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) + + logger.info "Forking repository from <#{from_path}> to <#{to_path}>." + cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) + + run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) + end + + def gitaly_fork_repository(new_shard_path, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) + rescue GRPC::BadStatus => e + logger.error "fork-repository failed: #{e.message}" + false + end end end end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index db532600d1b..d94082a3e30 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -10,6 +10,7 @@ module Gitlab DEFAULT_MODE = 0o100644 ACTIONS = %w(create create_dir update move delete).freeze + ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze attr_reader :repository, :raw_index @@ -20,6 +21,11 @@ module Gitlab delegate :read_tree, :get, to: :raw_index + def apply(action, options) + validate_action!(action) + public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend + end + def write_tree raw_index.write_tree(repository.rugged) end @@ -140,6 +146,12 @@ module Gitlab rescue Rugged::IndexError => e raise IndexError, e.message end + + def validate_action!(action) + unless ACTIONS.include?(action.to_s) + raise ArgumentError, "Unknown action '#{action}'" + end + end end end end diff --git a/lib/gitlab/git/info_attributes.rb b/lib/gitlab/git/info_attributes.rb new file mode 100644 index 00000000000..e79a440950b --- /dev/null +++ b/lib/gitlab/git/info_attributes.rb @@ -0,0 +1,49 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + +module Gitlab + module Git + # Parses gitattributes at `$GIT_DIR/info/attributes` + # + # Unlike Rugged this parser only needs a single IO call (a call to `open`), + # vastly reducing the time spent in extracting attributes. + # + # This class _only_ supports parsing the attributes file located at + # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files + # (`.gitattributes` is copied to this particular path). + # + # Basic usage: + # + # attributes = Gitlab::Git::InfoAttributes.new(some_repo.path) + # + # attributes.attributes('README.md') # => { "eol" => "lf } + class InfoAttributes + delegate :attributes, :patterns, to: :parser + + # path - The path to the Git repository. + def initialize(path) + @repo_path = File.expand_path(path) + end + + def parser + @parser ||= begin + if File.exist?(attributes_path) + File.open(attributes_path, 'r') do |file_handle| + AttributesParser.new(file_handle) + end + else + AttributesParser.new("") + end + end + end + + private + + def attributes_path + @attributes_path ||= File.join(@repo_path, 'info/attributes') + end + end + end +end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ef5bdbaf819..280def182d5 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -97,6 +97,11 @@ module Gitlab end end + def update_branch(branch_name, newrev, oldrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + end + private # Returns [newrev, should_run_after_create, should_run_after_create_branch] @@ -126,7 +131,10 @@ module Gitlab oldrev = branch.target - if oldrev == repository.merge_base(newrev, branch.target) + merge_base = repository.merge_base(newrev, branch.target) + raise Gitlab::Git::Repository::InvalidRef unless merge_base + + if oldrev == merge_base oldrev else raise Gitlab::Git::CommitError.new('Branch diverged') diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 42c80aabd0a..155cf52f050 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -6,7 +6,7 @@ module Gitlab class << self def normalize_path(filename) # Strip all leading slashes so that //foo -> foo - filename[/^\/*/] = '' + filename[%r{^/*}] = '' # Expand relative paths (e.g. foo/../bar) filename = Pathname.new(filename) diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index 1ccca13ce2f..e0bd2bbe47b 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -19,6 +19,8 @@ module Gitlab cmd_output = "" cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + stdout.set_encoding(Encoding::ASCII_8BIT) + yield(stdin) if block_given? stdin.close diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 372ce005b94..fa71a4e7ea7 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -23,7 +23,7 @@ module Gitlab # Ex. # Ref.extract_branch_name('refs/heads/master') #=> 'master' def self.extract_branch_name(str) - str.gsub(/\Arefs\/heads\//, '') + str.gsub(%r{\Arefs/heads/}, '') end # Gitaly: this method will probably be migrated indirectly via its call sites. @@ -33,9 +33,9 @@ module Gitlab object end - def initialize(repository, name, target, derefenced_target) + def initialize(repository, name, target, dereferenced_target) @name = Gitlab::Git.ref_name(name) - @dereferenced_target = derefenced_target + @dereferenced_target = dereferenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb new file mode 100644 index 00000000000..ebe46722890 --- /dev/null +++ b/lib/gitlab/git/remote_mirror.rb @@ -0,0 +1,89 @@ +module Gitlab + module Git + class RemoteMirror + def initialize(repository, ref_name) + @repository = repository + @ref_name = ref_name + end + + def update(only_branches_matching: []) + @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| + if is_enabled + gitaly_update(only_branches_matching) + else + rugged_update(only_branches_matching) + end + end + end + + private + + def gitaly_update(only_branches_matching) + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) + end + + def rugged_update(only_branches_matching) + local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) + remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) + + updated_branches = changed_refs(local_branches, remote_branches) + push_branches(updated_branches.keys) if updated_branches.present? + + delete_refs(local_branches, remote_branches) + + local_tags = refs_obj(@repository.tags) + remote_tags = refs_obj(@repository.remote_tags(@ref_name)) + + updated_tags = changed_refs(local_tags, remote_tags) + @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? + + delete_refs(local_tags, remote_tags) + end + + def refs_obj(refs, only_refs_matching: []) + refs.each_with_object({}) do |ref, refs| + next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) + + refs[ref.name] = ref + end + end + + def changed_refs(local_refs, remote_refs) + local_refs.select do |ref_name, ref| + remote_ref = remote_refs[ref_name] + + remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target + end + end + + def push_branches(branches) + default_branch, branches = branches.partition do |branch| + @repository.root_ref == branch + end + + # Push the default branch first so it works fine when remote mirror is empty. + branches.unshift(*default_branch) + + @repository.push_remote_branches(@ref_name, branches) + end + + def delete_refs(local_refs, remote_refs) + refs = refs_to_delete(local_refs, remote_refs) + + @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? + end + + def refs_to_delete(local_refs, remote_refs) + default_branch_id = @repository.commit.id + + remote_refs.select do |remote_ref_name, remote_ref| + next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo + + remote_ref_id = remote_ref.dereferenced_target.try(:id) + + remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 044c60caa05..ab1362a3bb0 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -21,6 +21,7 @@ module Gitlab REBASE_WORKTREE_PREFIX = 'rebase'.freeze SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze + GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -56,11 +57,12 @@ module Gitlab # Do nothing if hooks already exist unless real_local_hooks_path == File.realpath(global_hooks_path) - # Move the existing hooks somewhere safe - FileUtils.mv( - local_hooks_path, - "#{local_hooks_path}.old.#{Time.now.to_i}" - ) if File.exist?(local_hooks_path) + if File.exist?(local_hooks_path) + # Move the existing hooks somewhere safe + FileUtils.mv( + local_hooks_path, + "#{local_hooks_path}.old.#{Time.now.to_i}") + end # Create the hooks symlink FileUtils.ln_sf(global_hooks_path, local_hooks_path) @@ -82,7 +84,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -92,9 +94,15 @@ module Gitlab @gl_repository = gl_repository storage_path = Gitlab.config.repositories.storages[@storage]['path'] + @gitlab_projects = Gitlab::Git::GitlabProjects.new( + storage_path, + relative_path, + global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, + logger: Rails.logger + ) @path = File.join(storage_path, @relative_path) @name = @relative_path.split("/").last - @attributes = Gitlab::Git::Attributes.new(path) + @attributes = Gitlab::Git::InfoAttributes.new(path) end def ==(other) @@ -125,7 +133,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_repository_client.exists? else @@ -187,7 +195,7 @@ module Gitlab end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_migrate(:local_branches) do |is_enabled| if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else @@ -454,15 +462,18 @@ module Gitlab path: nil, follow: false, skip_merges: false, - disable_walk: false, after: nil, before: nil } options = default_options.merge(options) - options[:limit] ||= 0 options[:offset] ||= 0 + limit = options[:limit] + if limit == 0 || !limit.is_a?(Integer) + raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") + end + gitaly_migrate(:find_commits) do |is_enabled| if is_enabled gitaly_commit_client.find_commits(options) @@ -482,19 +493,17 @@ module Gitlab return [] end - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end + log_by_shell(sha, options) end def count_commits(options) + count_commits_options = process_count_commits_options(options) + gitaly_migrate(:count_commits) do |is_enabled| if is_enabled - count_commits_by_gitaly(options) + count_commits_by_gitaly(count_commits_options) else - count_commits_by_shelling_out(options) + count_commits_by_shelling_out(count_commits_options) end end end @@ -532,36 +541,57 @@ module Gitlab end # Counts the amount of commits between `from` and `to`. - def count_commits_between(from, to) - count_commits(ref: "#{from}..#{to}") + def count_commits_between(from, to, options = {}) + count_commits(from: from, to: to, **options) end # Returns the SHA of the most recent common ancestor of +from+ and +to+ - def merge_base_commit(from, to) + def merge_base(from, to) gitaly_migrate(:merge_base) do |is_enabled| if is_enabled gitaly_repository_client.find_merge_base(from, to) else - rugged.merge_base(from, to) + rugged_merge_base(from, to) end end end - alias_method :merge_base, :merge_base_commit # Gitaly note: JV: check gitlab-ee before removing this method. def rugged_is_ancestor?(ancestor_id, descendant_id) return false if ancestor_id.nil? || descendant_id.nil? - merge_base_commit(ancestor_id, descendant_id) == ancestor_id + rugged_merge_base(ancestor_id, descendant_id) == ancestor_id + rescue Rugged::OdbError + false end # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - gitaly_commit_client.ancestor?(from, to) + Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| + if is_enabled + gitaly_commit_client.ancestor?(from, to) + else + rugged_is_ancestor?(from, to) + end + end end def merged_branch_names(branch_names = []) - Set.new(git_merged_branch_names(branch_names)) + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + branches = gitaly_migrate(:merged_branch_names) do |is_enabled| + if is_enabled + gitaly_merged_branch_names(branch_names, root_sha) + else + git_merged_branch_names(branch_names, root_sha) + end + end + + Set.new(branches) end # Return an array of Diff objects that represent the diff @@ -588,46 +618,15 @@ module Gitlab if is_enabled gitaly_ref_client.find_ref_name(sha, ref_path) else - args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) # Not found -> ["", 0] # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - popen(args, @path).first.split.last + run_git(args).first.split.last end end end - # Returns branch names collection that contains the special commit(SHA1 - # or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branch_names_contains(commit) - branches_contains(commit).map { |c| c.name } - end - - # Returns branch collection that contains the special commit(SHA1 or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branches_contains(commit) - commit_obj = rugged.rev_parse(commit) - parent = commit_obj.parents.first unless commit_obj.parents.empty? - - walker = Rugged::Walker.new(rugged) - - rugged.branches.select do |branch| - walker.push(branch.target_id) - walker.hide(parent) if parent - result = walker.any? { |c| c.oid == commit_obj.oid } - walker.reset - - result - end - end - # Get refs hash which key is SHA1 # and value is a Rugged::Reference def refs_hash @@ -644,6 +643,7 @@ module Gitlab end end end + @refs_hash end @@ -680,11 +680,7 @@ module Gitlab if is_enabled gitaly_commit_client.commit_count(ref) else - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count + rugged_commit_count(ref) end end end @@ -887,17 +883,12 @@ module Gitlab end def delete_refs(*ref_names) - instructions = ref_names.map do |ref| - "delete #{ref}\x00\x00" - end - - command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - message, status = popen(command, path) do |stdin| - stdin.write(instructions.join) - end - - unless status.zero? - raise GitError.new("Could not delete refs #{ref_names}: #{message}") + gitaly_migrate(:delete_refs) do |is_enabled| + if is_enabled + gitaly_delete_refs(*ref_names) + else + git_delete_refs(*ref_names) + end end end @@ -918,31 +909,23 @@ module Gitlab # If `mirror_refmap` is present the remote is set as mirror with that mapping def add_remote(remote_name, url, mirror_refmap: nil) - rugged.remotes.create(remote_name, url) - - set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap - rescue Rugged::ConfigError - remote_update(remote_name, url: url) + gitaly_migrate(:remote_add_remote) do |is_enabled| + if is_enabled + gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) + else + rugged_add_remote(remote_name, url, mirror_refmap) + end + end end def remove_remote(remote_name) - # When a remote is deleted all its remote refs are deleted too, but in - # the case of mirrors we map its refs (that would usualy go under - # [remote_name]/) to the top level namespace. We clean the mapping so - # those don't get deleted. - if rugged.config["remote.#{remote_name}.mirror"] - rugged.config.delete("remote.#{remote_name}.fetch") + gitaly_migrate(:remote_remove_remote) do |is_enabled| + if is_enabled + gitaly_remote_client.remove_remote(remote_name) + else + rugged_remove_remote(remote_name) + end end - - rugged.remotes.delete(remote_name) - true - rescue Rugged::ConfigError - false - end - - # Returns true if a remote exists. - def remote_exists?(name) - rugged.remotes[name].present? end # Update the specified remote using the values in the +options+ hash @@ -1009,6 +992,18 @@ module Gitlab attributes(path)[name] end + # Check .gitattributes for a given ref + # + # This only checks the root .gitattributes file, + # it does not traverse subfolders to find additional .gitattributes files + # + # This method is around 30 times slower than `attributes`, + # which uses `$GIT_DIR/info/attributes` + def attributes_at(ref, file_path) + parser = AttributesAtRefParser.new(self, ref) + parser.attributes(file_path) + end + def languages(ref = nil) Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| if is_enabled @@ -1101,12 +1096,16 @@ module Gitlab end end - def write_ref(ref_path, ref) - raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') - raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + def write_ref(ref_path, ref, old_ref: nil, shell: true) + ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD" - input = "update #{ref_path}\x00#{ref}\x00\x00" - run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } + gitaly_migrate(:write_ref) do |is_enabled| + if is_enabled + gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell) + else + local_write_ref(ref_path, ref, old_ref: old_ref, shell: shell) + end + end end def fetch_ref(source_repository, source_ref:, target_ref:) @@ -1128,30 +1127,6 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args, chdir: path, env: {}, nice: false, &block) - cmd = [Gitlab.config.git.bin_path, *args] - cmd.unshift("nice") if nice - circuit_breaker.perform do - popen(cmd, chdir, env, &block) - end - end - - def run_git!(args, chdir: path, env: {}, nice: false, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) - - raise GitError, output unless status.zero? - - output - end - - # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) end @@ -1161,23 +1136,13 @@ module Gitlab end def fetch_repository_as_mirror(repository) - remote_name = "tmp-#{SecureRandom.hex}" - - # Notice that this feature flag is not for `fetch_repository_as_mirror` - # as a whole but for the fetching mechanism (file path or gitaly-ssh). - url, env = gitaly_migrate(:fetch_internal) do |is_enabled| + gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled| if is_enabled - repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) - [GITALY_INTERNAL_URL, repository.fetch_env] + gitaly_remote_client.fetch_internal_remote(repository) else - [repository.path, nil] + rugged_fetch_repository_as_mirror(repository) end end - - add_remote(remote_name, url, mirror_refmap: :all_refs) - fetch_remote(remote_name, env: env) - ensure - remove_remote(remote_name) end def blob_at(sha, path) @@ -1185,7 +1150,7 @@ module Gitlab end # Items should be of format [[commit_id, path], [commit_id1, path1]] - def batch_blobs(items, blob_size_limit: nil) + def batch_blobs(items, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) end @@ -1215,62 +1180,121 @@ module Gitlab end end - def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repository.path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + def create_from_bundle(bundle_path) + gitaly_migrate(:create_repo_from_bundle) do |is_enabled| + if is_enabled + gitaly_repository_client.create_from_bundle(bundle_path) + else + run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil) + self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) + end + end - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) + true + end - rebase_sha + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + gitaly_migrate(:rebase) do |is_enabled| + if is_enabled + gitaly_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + else + git_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end end end def rebase_in_progress?(rebase_id) - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + gitaly_migrate(:rebase_in_progress) do |is_enabled| + if is_enabled + gitaly_repository_client.rebase_in_progress?(rebase_id) + else + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + end end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) - env = git_env_for_user(user).merge( - 'GIT_AUTHOR_NAME' => author.name, - 'GIT_AUTHOR_EMAIL' => author.email - ) - diff_range = "#{start_sha}...#{end_sha}" - diff_files = run_git!( - %W(diff --name-only --diff-filter=a --binary #{diff_range}) - ).chomp + gitaly_migrate(:squash) do |is_enabled| + if is_enabled + gitaly_operation_client.user_squash(user, squash_id, branch, + start_sha, end_sha, author, message) + else + git_squash(user, squash_id, branch, start_sha, end_sha, author, message) + end + end + end - with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do - # Apply diff of the `diff_range` to the worktree - diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| - stdin.write(diff) + def squash_in_progress?(squash_id) + fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + end + + def push_remote_branches(remote_name, branch_names, forced: true) + success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + def bundle_to_disk(save_path) + gitaly_migrate(:bundle_to_disk) do |is_enabled| + if is_enabled + gitaly_repository_client.create_bundle(save_path) + else + run_git!(%W(bundle create #{save_path} --all)) end + end - # Commit the `diff_range` diff - run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + true + end - # Return the squash sha. May print a warning for ambiguous refs, but - # we can ignore that with `--quiet` and just take the SHA, if present. - # HEAD here always refers to the current HEAD commit, even if there is - # another ref called HEAD. - run_git!( - %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env - ).chomp + # rubocop:disable Metrics/ParameterLists + def multi_action( + user, branch_name:, message:, actions:, + author_email: nil, author_name: nil, + start_branch_name: nil, start_repository: self) + + gitaly_migrate(:operation_user_commit_files) do |is_enabled| + if is_enabled + gitaly_operation_client.user_commit_files(user, branch_name, + message, actions, author_email, author_name, + start_branch_name, start_repository) + else + rugged_multi_action(user, branch_name, message, actions, + author_email, author_name, start_branch_name, start_repository) + end end end + # rubocop:enable Metrics/ParameterLists - def squash_in_progress?(squash_id) - fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + def write_config(full_path:) + return unless full_path.present? + + gitaly_migrate(:write_config) do |is_enabled| + if is_enabled + gitaly_repository_client.write_config(full_path: full_path) + else + rugged_write_config(full_path: full_path) + end + end end def gitaly_repository @@ -1297,6 +1321,18 @@ module Gitlab @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) end + def gitaly_remote_client + @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) + end + + def gitaly_blob_client + @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self) + end + + def gitaly_conflicts_client(our_commit_oid, their_commit_oid) + Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e @@ -1307,8 +1343,166 @@ module Gitlab raise CommandError.new(e) end + def branch_names_contains_sha(sha) + gitaly_migrate(:branch_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names_contains_sha(sha) + else + refs_contains_sha(:branch, sha) + end + end + end + + def tag_names_contains_sha(sha) + gitaly_migrate(:tag_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names_contains_sha(sha) + else + refs_contains_sha(:tag, sha) + end + end + end + + def search_files_by_content(query, ref) + return [] if empty? || query.blank? + + offset = 2 + args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) + + run_git(args).first.scrub.split(/^--$/) + end + + def can_be_merged?(source_sha, target_branch) + gitaly_migrate(:can_be_merged) do |is_enabled| + if is_enabled + gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target) + else + rugged_can_be_merged?(source_sha, target_branch) + end + end + end + + def search_files_by_name(query, ref) + safe_query = Regexp.escape(query.sub(%r{^/*}, "")) + + return [] if empty? || safe_query.blank? + + args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{safe_query}) + + run_git(args).first.lines.map(&:strip) + end + + def find_commits_by_message(query, ref, path, limit, offset) + gitaly_migrate(:commits_by_message) do |is_enabled| + if is_enabled + find_commits_by_message_by_gitaly(query, ref, path, limit, offset) + else + find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) + end + end + end + + def shell_blame(sha, path) + output, _status = run_git(%W(blame -p #{sha} -- #{path})) + output + end + + def can_be_merged?(source_sha, target_branch) + gitaly_migrate(:can_be_merged) do |is_enabled| + if is_enabled + gitaly_can_be_merged?(source_sha, find_branch(target_branch).target) + else + rugged_can_be_merged?(source_sha, target_branch) + end + end + end + + def last_commit_id_for_path(sha, path) + gitaly_migrate(:last_commit_for_path) do |is_enabled| + if is_enabled + last_commit_for_path_by_gitaly(sha, path).id + else + last_commit_id_for_path_by_shelling_out(sha, path) + end + end + end + private + def local_write_ref(ref_path, ref, old_ref: nil, shell: true) + if shell + shell_write_ref(ref_path, ref, old_ref) + else + rugged_write_ref(ref_path, ref) + end + end + + def refs_contains_sha(ref_type, sha) + args = %W(#{ref_type} --contains #{sha}) + names = run_git(args).first + + return [] unless names.respond_to?(:split) + + names = names.split("\n").map(&:strip) + + names.each do |name| + name.slice! '* ' + end + + names + end + + def rugged_write_config(full_path:) + rugged.config['gitlab.fullpath'] = full_path + end + + def shell_write_ref(ref_path, ref, old_ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") + + input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" + run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } + end + + def rugged_write_ref(ref_path, ref) + rugged.references.create(ref_path, ref, force: true) + rescue Rugged::ReferenceError => ex + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + rescue Rugged::OSError => ex + raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ + + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + end + + def run_git(args, chdir: path, env: {}, nice: false, &block) + cmd = [Gitlab.config.git.bin_path, *args] + cmd.unshift("nice") if nice + + object_directories = alternate_object_directories + if object_directories.any? + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) + end + + circuit_breaker.perform do + popen(cmd, chdir, env, &block) + end + end + + def run_git!(args, chdir: path, env: {}, nice: false, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + + raise GitError, output unless status.zero? + + output + end + + def run_git_with_timeout(args, timeout, env: {}) + circuit_breaker.perform do + popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) + end + end + def fresh_worktree?(path) File.exist?(path) && !clean_stuck_worktree(path) end @@ -1323,7 +1517,7 @@ module Gitlab if sparse_checkout_files # Create worktree without checking out run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp configure_sparse_checkout(worktree_git_path, sparse_checkout_files) @@ -1414,14 +1608,7 @@ module Gitlab sort_branches(branches, sort_by) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 - def git_merged_branch_names(branch_names = []) - return [] unless root_ref - - root_sha = find_branch(root_ref)&.target - - return [] unless root_sha - + def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} --format=%(refname:short)\ %(objectname)] + branch_names @@ -1435,22 +1622,32 @@ module Gitlab end end - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] + def gitaly_merged_branch_names(branch_names, root_sha) + qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } + + gitaly_ref_client.merged_branches(qualified_branch_names) + .reject { |b| b.target == root_sha } + .map(&:name) end - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a + def process_count_commits_options(options) + if options[:from] || options[:to] + ref = + if options[:left_right] # Compare with merge-base for left-right + "#{options[:from]}...#{options[:to]}" + else + "#{options[:from]}..#{options[:to]}" + end + + options.merge(ref: ref) + + elsif options[:ref] && options[:left_right] + from, to = options[:ref].match(/\A([^\.]*)\.{2,3}([^\.]*)\z/)[1..2] + + options.merge(from: from, to: to) + else + options + end end # Gitaly note: JV: although #log_by_shell shells out to Git I think the @@ -1466,7 +1663,7 @@ module Gitlab offset_in_ruby = use_follow_flag && options[:offset].present? limit += offset if offset_in_ruby - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd = %w[log] cmd << "--max-count=#{limit}" cmd << '--format=%H' cmd << "--skip=#{offset}" unless offset_in_ruby @@ -1482,7 +1679,7 @@ module Gitlab cmd += Array(options[:path]) end - raw_output = IO.popen(cmd) { |io| io.read } + raw_output, _status = run_git(cmd) lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } @@ -1520,18 +1717,23 @@ module Gitlab end def alternate_object_directories - relative_paths = Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + relative_paths = relative_object_directories if relative_paths.any? relative_paths.map { |d| File.join(path, d) } else - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES) - .flatten - .compact - .flat_map { |d| d.split(File::PATH_SEPARATOR) } + absolute_object_directories.flat_map { |d| d.split(File::PATH_SEPARATOR) } end end + def relative_object_directories + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + end + + def absolute_object_directories + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).flatten.compact + end + # Get the content of a blob for a given commit. If the blob is a commit # (for submodules) then return the blob's OID. def blob_content(commit, blob_name) @@ -1657,19 +1859,59 @@ module Gitlab end def count_commits_by_gitaly(options) - gitaly_commit_client.commit_count(options[:ref], options) + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] + else + gitaly_commit_client.commit_count(options[:ref], options) + end end def count_commits_by_shelling_out(options) - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] + cmd = count_commits_shelling_command(options) + + raw_output, _status = run_git(cmd) + + process_count_commits_raw_output(raw_output, options) + end + + def count_commits_shelling_command(options) + cmd = %w[rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << "--max-count=#{options[:max_count]}" if options[:max_count] + cmd << "--left-right" if options[:left_right] cmd += %W[--count #{options[:ref]}] cmd += %W[-- #{options[:path]}] if options[:path].present? + cmd + end - raw_output = IO.popen(cmd) { |io| io.read } + def process_count_commits_raw_output(raw_output, options) + if options[:left_right] + result = raw_output.scan(/\d+/).map(&:to_i) - raw_output.to_i + if result.sum != options[:max_count] + result + else # Reaching max count, right is not accurate + right_option = + process_count_commits_options(options + .except(:left_right, :from, :to) + .merge(ref: options[:to])) + + right = count_commits_by_shelling_out(right_option) + + [result.first, right] # left should be accurate in the first call + end + else + raw_output.to_i + end end def gitaly_ls_files(ref) @@ -1686,20 +1928,17 @@ module Gitlab return [] end - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) - cmd += %w(-r) - cmd += %w(--full-tree) - cmd += %w(--full-name) - cmd += %W(-- #{actual_ref}) + cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) + raw_output, _status = run_git(cmd) - raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + lines = raw_output.split("\n").map do |f| stuff, path = f.split("\t") _mode, type, _sha = stuff.split(" ") path if type == "blob" # Contain only blob type end - raw_output.compact + lines.compact end # Returns true if the given ref name exists @@ -1773,7 +2012,7 @@ module Gitlab target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'} raise InvalidRef.new("Invalid reference #{start_point}") end @@ -1889,6 +2128,71 @@ module Gitlab tree_id end + def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end + + def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + if remote_repository.is_a?(RemoteRepository) + env.merge!(remote_repository.fetch_env) + remote_repo_path = GITALY_INTERNAL_URL + else + remote_repo_path = remote_repository.path + end + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repo_path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + + def git_squash(user, squash_id, branch, start_sha, end_sha, author, message) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -1912,13 +2216,162 @@ module Gitlab source_sha end - rescue Rugged::ReferenceError + rescue Rugged::ReferenceError, InvalidRef raise ArgumentError, 'Invalid merge source' end + def rugged_add_remote(remote_name, url, mirror_refmap) + rugged.remotes.create(remote_name, url) + + set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap + rescue Rugged::ConfigError + remote_update(remote_name, url: url) + end + + def git_delete_refs(*ref_names) + instructions = ref_names.map do |ref| + "delete #{ref}\x00\x00" + end + + message, status = run_git(%w[update-ref --stdin -z]) do |stdin| + stdin.write(instructions.join) + end + + unless status.zero? + raise GitError.new("Could not delete refs #{ref_names}: #{message}") + end + end + + def gitaly_delete_refs(*ref_names) + gitaly_ref_client.delete_refs(refs: ref_names) + end + + def rugged_remove_remote(remote_name) + # When a remote is deleted all its remote refs are deleted too, but in + # the case of mirrors we map its refs (that would usualy go under + # [remote_name]/) to the top level namespace. We clean the mapping so + # those don't get deleted. + if rugged.config["remote.#{remote_name}.mirror"] + rugged.config.delete("remote.#{remote_name}.fetch") + end + + rugged.remotes.delete(remote_name) + true + rescue Rugged::ConfigError + false + end + + def rugged_fetch_repository_as_mirror(repository) + remote_name = "tmp-#{SecureRandom.hex}" + repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) + + add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) + fetch_remote(remote_name, env: repository.fetch_env) + ensure + remove_remote(remote_name) + end + + def rugged_multi_action( + user, branch_name, message, actions, author_email, author_name, + start_branch_name, start_repository) + + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + index = Gitlab::Git::Index.new(self) + parents = [] + + if start_commit + index.read_tree(start_commit.rugged_commit.tree) + parents = [start_commit.sha] + end + + actions.each { |opts| index.apply(opts.delete(:action), opts) } + + committer = user_to_committer(user) + author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer + options = { + tree: index.write_tree, + message: message, + parents: parents, + author: author, + committer: committer + } + + create_commit(options) + end + end + def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end + + def gitaly_can_be_merged?(their_commit, our_commit) + !gitaly_conflicts_client(our_commit, their_commit).conflicts? + end + + def rugged_can_be_merged?(their_commit, our_commit) + !rugged.merge_commits(our_commit, their_commit).conflicts? + end + + def gitlab_projects_error + raise CommandError, @gitlab_projects.output + end + + def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) + ref ||= root_ref + + args = %W( + log #{ref} --pretty=%H --skip #{offset} + --max-count #{limit} --grep=#{query} --regexp-ignore-case + ) + args = args.concat(%W(-- #{path})) if path.present? + + git_log_results = run_git(args).first.lines + + git_log_results.map { |c| commit(c.chomp) }.compact + end + + def find_commits_by_message_by_gitaly(query, ref, path, limit, offset) + gitaly_commit_client + .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset) + .map { |c| commit(c) } + end + + def gitaly_can_be_merged?(their_commit, our_commit) + !gitaly_conflicts_client(our_commit, their_commit).conflicts? + end + + def rugged_can_be_merged?(their_commit, our_commit) + !rugged.merge_commits(our_commit, their_commit).conflicts? + end + + def last_commit_for_path_by_gitaly(sha, path) + gitaly_commit_client.last_commit_for_path(sha, path) + end + + def last_commit_id_for_path_by_shelling_out(sha, path) + args = %W(rev-list --max-count=1 #{sha} -- #{path}) + run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip + end + + def rugged_merge_base(from, to) + rugged.merge_base(from, to) + rescue Rugged::ReferenceError + nil + end + + def rugged_commit_count(ref) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + rescue Rugged::ReferenceError + 0 + end end end end diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index effb1f0ca19..dc424a433fb 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -43,7 +43,7 @@ module Gitlab branches = [] rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| - name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '') + name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '') begin target_commit = Gitlab::Git::Commit.find(self, ref.target) diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 4974205b8fd..f8b2e7e0e21 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -95,7 +95,7 @@ module Gitlab object_output.map do |output_line| sha, path = output_line.split(' ', 2) - next if require_path && path.blank? + next if require_path && path.to_s.empty? sha end.reject(&:nil?) diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 1307f400700..0a4e557b59b 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -27,6 +27,7 @@ module Gitlab status = nil while status.nil? + if deadline > Time.now.utc sleep(wait_time) _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index bc4e160dce9..8a8f7b051ed 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. -# module Gitlab module Git class Tag < Ref diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 5cf336af3c6..ba6058fd3c9 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -83,6 +83,8 @@ module Gitlab commit_id: sha ) end + rescue Rugged::ReferenceError + [] end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index d4a53d32c28..daa17fc72cf 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -25,8 +25,9 @@ module Gitlab @repository.exists? end + # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539 def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) gollum_wiki.clear_cache @@ -47,8 +48,9 @@ module Gitlab end end + # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094 def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) gollum_wiki.clear_cache @@ -68,8 +70,9 @@ module Gitlab end end + # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42039 def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_find_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_find_page(title: title, version: version, dir: dir) else @@ -117,6 +120,20 @@ module Gitlab page.url_path end + def page_formatted_data(title:, dir: nil, version: nil) + version = version&.id + + @repository.gitaly_migrate(:wiki_page_formatted_data) do |is_enabled| + if is_enabled + gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) + else + # We don't use #page because if wiki_find_page feature is enabled, we would + # get a page without formatted_data. + gollum_find_page(title: title, dir: dir, version: version)&.formatted_data + end + end + end + private # options: @@ -178,7 +195,10 @@ module Gitlab assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - gollum_wiki.write_page(name, format, content, commit_details.to_h) + filename = File.basename(name) + dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir + + gollum_wiki.write_page(filename, format, content, commit_details.to_h, dir) nil rescue Gollum::DuplicatePageError => e @@ -196,7 +216,15 @@ module Gitlab assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + page = gollum_page_by_path(page_path) + committer = Gollum::Committer.new(page.wiki, commit_details.to_h) + + # Instead of performing two renames if the title has changed, + # the update_page will only update the format and content and + # the rename_page will do anything related to moving/renaming + gollum_wiki.update_page(page, page.name, format, content, committer: committer) + gollum_wiki.rename_page(page, title, committer: committer) + committer.commit nil end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index a06bac4414f..669ae11a423 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -1,7 +1,7 @@ module Gitlab module Git class WikiPage - attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data # This class is meant to be serializable so that it can be constructed # by Gitaly and sent over the network to GitLab. @@ -21,6 +21,7 @@ module Gitlab @raw_data = gollum_page.raw_data @name = gollum_page.name @historical = gollum_page.historical? + @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page) @version = version end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index b753ac46291..c5d3e944f7d 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -1,9 +1,12 @@ require 'base64' require 'gitaly' +require 'grpc/health/v1/health_pb' +require 'grpc/health/v1/health_services_pb' module Gitlab module GitalyClient + include Gitlab::Metrics::Methods module MigrationStatus DISABLED = 1 OPT_IN = 2 @@ -31,8 +34,6 @@ module Gitlab CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new - METRICS_MUTEX = Mutex.new - private_constant :MUTEX, :METRICS_MUTEX class << self attr_accessor :query_time @@ -40,28 +41,14 @@ module Gitlab self.query_time = 0 - def self.migrate_histogram - @migrate_histogram ||= - METRICS_MUTEX.synchronize do - # If a thread was blocked on the mutex, the value was set already - return @migrate_histogram if @migrate_histogram - - Gitlab::Metrics.histogram(:gitaly_migrate_call_duration_seconds, - "Gitaly migration call execution timings", - gitaly_enabled: nil, feature: nil) - end + define_histogram :gitaly_migrate_call_duration_seconds do + docstring "Gitaly migration call execution timings" + base_labels gitaly_enabled: nil, feature: nil end - def self.gitaly_call_histogram - @gitaly_call_histogram ||= - METRICS_MUTEX.synchronize do - # If a thread was blocked on the mutex, the value was set already - return @gitaly_call_histogram if @gitaly_call_histogram - - Gitlab::Metrics.histogram(:gitaly_controller_action_duration_seconds, - "Gitaly endpoint histogram by controller and action combination", - Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)) - end + define_histogram :gitaly_controller_action_duration_seconds do + docstring "Gitaly endpoint histogram by controller and action combination" + base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil) end def self.stub(name, storage) @@ -69,14 +56,27 @@ module Gitlab @stubs ||= {} @stubs[storage] ||= {} @stubs[storage][name] ||= begin - klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) - addr = address(storage) - addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + klass = stub_class(name) + addr = stub_address(storage) klass.new(addr, :this_channel_is_insecure) end end end + def self.stub_class(name) + if name == :health_check + Grpc::Health::V1::Health::Stub + else + Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + end + end + + def self.stub_address(storage) + addr = address(storage) + addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + addr + end + def self.clear_stubs! MUTEX.synchronize do @stubs = nil @@ -130,7 +130,7 @@ module Gitlab # Keep track, seperately, for the performance bar self.query_time += duration - gitaly_call_histogram.observe( + gitaly_controller_action_duration_seconds.observe( current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), duration) end @@ -232,7 +232,7 @@ module Gitlab yield is_enabled ensure total_time = Gitlab::Metrics::System.monotonic_time - start - migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) + gitaly_migrate_call_duration_seconds.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) feature_stack.shift Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? end @@ -330,22 +330,6 @@ module Gitlab Google::Protobuf::Timestamp.new(seconds: t.to_i) end - def self.encode(s) - return "" if s.nil? - - s.dup.force_encoding(Encoding::ASCII_8BIT) - end - - def self.binary_stringio(s) - io = StringIO.new(s || '') - io.set_encoding(Encoding::ASCII_8BIT) - io - end - - def self.encode_repeated(a) - Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) - end - # The default timeout on all Gitaly calls def self.default_timeout return 0 if Sidekiq.server? diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index a250eb75bd4..d70a1a7665e 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -32,6 +32,28 @@ module Gitlab binary: Gitlab::Git::Blob.binary?(data) ) end + + def batch_lfs_pointers(blob_ids) + return [] if blob_ids.empty? + + request = Gitaly::GetLFSPointersRequest.new( + repository: @gitaly_repo, + blob_ids: blob_ids + ) + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) + + response.flat_map do |message| + message.lfs_pointers.map do |lfs_pointer| + Gitlab::Git::Blob.new( + id: lfs_pointer.oid, + size: lfs_pointer.size, + data: lfs_pointer.data, + binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) + ) + end + end + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index fb3e27770b4..5767f06b0ce 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class CommitService + include Gitlab::EncodingHelper + # The ID of empty tree. # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze @@ -13,7 +15,7 @@ module Gitlab def ls_files(revision) request = Gitaly::ListFilesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) @@ -36,19 +38,27 @@ module Gitlab from_id = case from when NilClass EMPTY_TREE_ID - when Rugged::Commit - from.oid else - from + if from.respond_to?(:oid) + # This is meant to match a Rugged::Commit. This should be impossible in + # the future. + from.oid + else + from + end end to_id = case to when NilClass EMPTY_TREE_ID - when Rugged::Commit - to.oid else - to + if to.respond_to?(:oid) + # This is meant to match a Rugged::Commit. This should be impossible in + # the future. + to.oid + else + to + end end request_params = diff_between_commits_request_params(from_id, to_id, options) @@ -73,7 +83,7 @@ module Gitlab request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: ref, - path: GitalyClient.encode(path), + path: encode_binary(path), limit: limit.to_i ) @@ -98,8 +108,8 @@ module Gitlab def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: path.present? ? GitalyClient.encode(path) : '.' + revision: encode_binary(revision), + path: path.present? ? encode_binary(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) @@ -112,8 +122,8 @@ module Gitlab type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), name: File.basename(gitaly_tree_entry.path), - path: GitalyClient.encode(gitaly_tree_entry.path), - flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path), + path: encode_binary(gitaly_tree_entry.path), + flat_path: encode_binary(gitaly_tree_entry.flat_path), commit_id: gitaly_tree_entry.commit_oid ) end @@ -123,11 +133,12 @@ module Gitlab def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, - revision: ref + revision: encode_binary(ref) ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? - request.path = options[:path] if options[:path].present? + request.path = encode_binary(options[:path]) if options[:path].present? + request.max_count = options[:max_count] if options[:max_count].present? GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end @@ -135,8 +146,8 @@ module Gitlab def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path.to_s) + revision: encode_binary(revision), + path: encode_binary(path.to_s) ) gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit @@ -174,7 +185,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) - rescue GRPC::Unknown # If no repository is found, happens mainly during testing + rescue GRPC::NotFound # If no repository is found, happens mainly during testing [] end @@ -202,8 +213,8 @@ module Gitlab def raw_blame(revision, path) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path) + revision: encode_binary(revision), + path: encode_binary(path) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) @@ -213,7 +224,7 @@ module Gitlab def find_commit(revision) request = Gitaly::FindCommitRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) @@ -224,7 +235,7 @@ module Gitlab def patch(revision) request = Gitaly::CommitPatchRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) @@ -234,7 +245,7 @@ module Gitlab def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end @@ -246,13 +257,13 @@ module Gitlab offset: options[:offset], follow: options[:follow], skip_merges: options[:skip_merges], - disable_walk: options[:disable_walk] + disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] - request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + request.revision = encode_binary(options[:ref]) if options[:ref] - request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + request.paths = encode_repeated(Array(options[:path])) if options[:path].present? response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) @@ -264,7 +275,7 @@ module Gitlab enum = Enumerator.new do |y| shas.each_slice(20) do |revs| - request.shas = GitalyClient.encode_repeated(revs) + request.shas = encode_repeated(revs) y.yield request @@ -279,6 +290,23 @@ module Gitlab end end + def extract_signature(commit_id) + request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id) + response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request) + + signature = ''.b + signed_text = ''.b + + response.each do |message| + signature << message.signature + signed_text << message.signed_text + end + + return if signature.blank? && signed_text.blank? + + [signature, signed_text] + end + private def call_commit_diff(request_params, options = {}) @@ -303,7 +331,7 @@ module Gitlab repository: @gitaly_repo, left_commit_id: from_id, right_commit_id: to_id, - paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } + paths: options.fetch(:paths, []).compact.map { |path| encode_binary(path) } } end @@ -314,6 +342,10 @@ module Gitlab end end end + + def encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) + end end end end diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb new file mode 100644 index 00000000000..97c13d1fdb0 --- /dev/null +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -0,0 +1,47 @@ +module Gitlab + module GitalyClient + class ConflictFilesStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_file = nil + + @rpc_response.each do |msg| + msg.files.each do |gitaly_file| + if gitaly_file.header + yield current_file if current_file + + current_file = file_from_gitaly_header(gitaly_file.header) + else + current_file.content << gitaly_file.content + end + end + end + + yield current_file if current_file + end + + private + + def file_from_gitaly_header(header) + Gitlab::Git::Conflict::File.new( + Gitlab::GitalyClient::Util.git_repository(header.repository), + header.commit_oid, + conflict_from_gitaly_file_header(header), + '' + ) + end + + def conflict_from_gitaly_file_header(header) + { + ours: { path: header.our_path, mode: header.our_mode }, + theirs: { path: header.their_path } + } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb new file mode 100644 index 00000000000..e14734495a8 --- /dev/null +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -0,0 +1,72 @@ +module Gitlab + module GitalyClient + class ConflictsService + include Gitlab::EncodingHelper + + MAX_MSG_SIZE = 128.kilobytes.freeze + + def initialize(repository, our_commit_oid, their_commit_oid) + @gitaly_repo = repository.gitaly_repository + @repository = repository + @our_commit_oid = our_commit_oid + @their_commit_oid = their_commit_oid + end + + def list_conflict_files + request = Gitaly::ListConflictFilesRequest.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + their_commit_oid: @their_commit_oid + ) + response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request) + + GitalyClient::ConflictFilesStitcher.new(response) + end + + def conflicts? + list_conflict_files.any? + rescue GRPC::FailedPrecondition + # The server raises this exception when it encounters ConflictSideMissing, which + # means a conflict exists but its `theirs` or `ours` data is nil due to a non-existent + # file in one of the trees. + true + end + + def resolve_conflicts(target_repository, resolution, source_branch, target_branch) + reader = binary_stringio(resolution.files.to_json) + + req_enum = Enumerator.new do |y| + header = resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + y.yield Gitaly::ResolveConflictsRequest.new(header: header) + + until reader.eof? + chunk = reader.read(MAX_MSG_SIZE) + + y.yield Gitaly::ResolveConflictsRequest.new(files_json: chunk) + end + end + + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage) + + if response.resolution_error.present? + raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error + end + end + + private + + def resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + Gitaly::ResolveConflictsRequestHeader.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + target_repository: target_repository.gitaly_repository, + their_commit_oid: @their_commit_oid, + source_branch: source_branch, + target_branch: target_branch, + commit_message: resolution.commit_message, + user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/health_check_service.rb b/lib/gitlab/gitaly_client/health_check_service.rb new file mode 100644 index 00000000000..6c1213f5e20 --- /dev/null +++ b/lib/gitlab/gitaly_client/health_check_service.rb @@ -0,0 +1,19 @@ +module Gitlab + module GitalyClient + class HealthCheckService + def initialize(storage) + @storage = storage + end + + # Sends a gRPC health ping to the Gitaly server for the storage shard. + def check + request = Grpc::Health::V1::HealthCheckRequest.new + response = GitalyClient.call(@storage, :health_check, :check, request, timeout: GitalyClient.fast_timeout) + + { success: response&.status == :SERVING } + rescue GRPC::BadStatus => e + { success: false, message: e.to_s } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 400a4af363b..831cfd1e014 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -1,6 +1,10 @@ module Gitlab module GitalyClient class OperationService + include Gitlab::EncodingHelper + + MAX_MSG_SIZE = 128.kilobytes.freeze + def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -9,7 +13,7 @@ module Gitlab def rm_tag(tag_name, user) request = Gitaly::UserDeleteTagRequest.new( repository: @gitaly_repo, - tag_name: GitalyClient.encode(tag_name), + tag_name: encode_binary(tag_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -24,9 +28,9 @@ module Gitlab request = Gitaly::UserCreateTagRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - tag_name: GitalyClient.encode(tag_name), - target_revision: GitalyClient.encode(target), - message: GitalyClient.encode(message.to_s) + tag_name: encode_binary(tag_name), + target_revision: encode_binary(target), + message: encode_binary(message.to_s) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) @@ -44,12 +48,13 @@ module Gitlab def user_create_branch(branch_name, user, start_point) request = Gitaly::UserCreateBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - start_point: GitalyClient.encode(start_point) + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_branch, request) + if response.pre_receive_error.present? raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) end @@ -64,7 +69,7 @@ module Gitlab def user_delete_branch(branch_name, user) request = Gitaly::UserDeleteBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -89,8 +94,8 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch), - message: GitalyClient.encode(message) + branch: encode_binary(target_branch), + message: encode_binary(message) ) ) @@ -98,7 +103,14 @@ module Gitlab request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) - branch_update = response_enum.next.branch_update + second_response = response_enum.next + + if second_response.pre_receive_error.present? + raise Gitlab::Git::HooksService::PreReceiveError, second_response.pre_receive_error + end + + branch_update = second_response.branch_update + return if branch_update.nil? raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) @@ -111,7 +123,7 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch) + branch: encode_binary(target_branch) ) branch_update = GitalyClient.call( @@ -143,6 +155,103 @@ module Gitlab start_repository: start_repository) end + def user_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + request = Gitaly::UserRebaseRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + rebase_id: rebase_id.to_s, + branch: encode_binary(branch), + branch_sha: branch_sha, + remote_repository: remote_repository.gitaly_repository, + remote_branch: encode_binary(remote_branch) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_rebase, + request, + remote_storage: remote_repository.storage + ) + + if response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + elsif response.git_error.presence + raise Gitlab::Git::Repository::GitError, response.git_error + else + response.rebase_sha + end + end + + def user_squash(user, squash_id, branch, start_sha, end_sha, author, message) + request = Gitaly::UserSquashRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + squash_id: squash_id.to_s, + branch: encode_binary(branch), + start_sha: start_sha, + end_sha: end_sha, + author: Gitlab::Git::User.from_gitlab(author).to_gitaly, + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_squash, + request + ) + + if response.git_error.presence + raise Gitlab::Git::Repository::GitError, response.git_error + end + + response.squash_sha + end + + def user_commit_files( + user, branch_name, commit_message, actions, author_email, author_name, + start_branch_name, start_repository) + + req_enum = Enumerator.new do |y| + header = user_commit_files_request_header(user, branch_name, + commit_message, actions, author_email, author_name, + start_branch_name, start_repository) + + y.yield Gitaly::UserCommitFilesRequest.new(header: header) + + actions.each do |action| + action_header = user_commit_files_action_header(action) + y.yield Gitaly::UserCommitFilesRequest.new( + action: Gitaly::UserCommitFilesAction.new(header: action_header) + ) + + reader = binary_stringio(action[:content]) + + until reader.eof? + chunk = reader.read(MAX_MSG_SIZE) + + y.yield Gitaly::UserCommitFilesRequest.new( + action: Gitaly::UserCommitFilesAction.new(content: chunk) + ) + end + end + end + + response = GitalyClient.call(@repository.storage, :operation_service, + :user_commit_files, req_enum, remote_storage: start_repository.storage) + + if (pre_receive_error = response.pre_receive_error.presence) + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + + if (index_error = response.index_error.presence) + raise Gitlab::Git::Index::IndexError, index_error + end + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + private def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) @@ -152,9 +261,9 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit: commit.to_gitaly_commit, - branch_name: GitalyClient.encode(branch_name), - message: GitalyClient.encode(message), - start_branch_name: GitalyClient.encode(start_branch_name.to_s), + branch_name: encode_binary(branch_name), + message: encode_binary(message), + start_branch_name: encode_binary(start_branch_name.to_s), start_repository: start_repository.gitaly_repository ) @@ -180,6 +289,33 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end end + + def user_commit_files_request_header( + user, branch_name, commit_message, actions, author_email, author_name, + start_branch_name, start_repository) + + Gitaly::UserCommitFilesRequestHeader.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + branch_name: encode_binary(branch_name), + commit_message: encode_binary(commit_message), + commit_author_name: encode_binary(author_name), + commit_author_email: encode_binary(author_email), + start_branch_name: encode_binary(start_branch_name), + start_repository: start_repository.gitaly_repository + ) + end + + def user_commit_files_action_header(action) + Gitaly::UserCommitFilesActionHeader.new( + action: action[:action].upcase.to_sym, + file_path: encode_binary(action[:file_path]), + previous_path: encode_binary(action[:previous_path]), + base64_content: action[:encoding] == 'base64' + ) + rescue RangeError + raise ArgumentError, "Unknown action '#{action[:action]}'" + end end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 066e4e183c0..ba6b577fd17 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -14,12 +14,18 @@ module Gitlab request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) - response.flat_map do |message| - message.branches.map do |branch| - target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) - Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) - end - end + consume_find_all_branches_response(response) + end + + def merged_branches(branch_names = []) + request = Gitaly::FindAllBranchesRequest.new( + repository: @gitaly_repo, + merged_only: true, + merged_branches: branch_names.map { |s| encode_binary(s) } + ) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + consume_find_all_branches_response(response) end def default_branch_name @@ -62,7 +68,7 @@ module Gitlab request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) - consume_branches_response(response) + consume_find_local_branches_response(response) end def tags @@ -72,7 +78,7 @@ module Gitlab end def ref_exists?(ref_name) - request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name)) + request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) response.value rescue GRPC::InvalidArgument => e @@ -82,7 +88,7 @@ module Gitlab def find_branch(branch_name) request = Gitaly::FindBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) @@ -96,8 +102,8 @@ module Gitlab def create_branch(ref, start_point) request = Gitaly::CreateBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(ref), - start_point: GitalyClient.encode(start_point) + name: encode_binary(ref), + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) @@ -121,19 +127,48 @@ module Gitlab def delete_branch(branch_name) request = Gitaly::DeleteBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) end - def delete_refs(except_with_prefixes:) + def delete_refs(refs: [], except_with_prefixes: []) request = Gitaly::DeleteRefsRequest.new( repository: @gitaly_repo, - except_with_prefix: except_with_prefixes + refs: refs.map { |r| encode_binary(r) }, + except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + + raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? + end + + # Limit: 0 implies no limit, thus all tag names will be returned + def tag_names_contains_sha(sha, limit: 0) + request = Gitaly::ListTagNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit ) - GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :tag_names) + end + + # Limit: 0 implies no limit, thus all tag names will be returned + def branch_names_contains_sha(sha, limit: 0) + request = Gitaly::ListBranchNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :branch_names) end private @@ -151,7 +186,7 @@ module Gitlab enum_value end - def consume_branches_response(response) + def consume_find_local_branches_response(response) response.flat_map do |message| message.branches.map do |gitaly_branch| Gitlab::Git::Branch.new( @@ -164,6 +199,15 @@ module Gitlab end end + def consume_find_all_branches_response(response) + response.flat_map do |message| + message.branches.map do |branch| + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + def consume_tags_response(response) response.flat_map do |message| message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } @@ -197,6 +241,13 @@ module Gitlab Gitlab::Git::Commit.decorate(@repository, hash) end + def consume_ref_contains_sha_response(stream, collection_name) + stream.each_with_object([]) do |response, array| + encoded_names = response.send(collection_name).map { |b| Gitlab::Git.ref_name(b) } # rubocop:disable GitlabSecurity/PublicSend + array.concat(encoded_names) + end + end + def invalid_ref!(message) raise Gitlab::Git::Repository::InvalidRef.new(message) end diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb new file mode 100644 index 00000000000..58c356edfd1 --- /dev/null +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -0,0 +1,70 @@ +module Gitlab + module GitalyClient + class RemoteService + MAX_MSG_SIZE = 128.kilobytes.freeze + + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def add_remote(name, url, mirror_refmaps) + request = Gitaly::AddRemoteRequest.new( + repository: @gitaly_repo, + name: name, + url: url, + mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s) + ) + + GitalyClient.call(@storage, :remote_service, :add_remote, request) + end + + def remove_remote(name) + request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) + + response = GitalyClient.call(@storage, :remote_service, :remove_remote, request) + + response.result + end + + def fetch_internal_remote(repository) + request = Gitaly::FetchInternalRemoteRequest.new( + repository: @gitaly_repo, + remote_repository: repository.gitaly_repository + ) + + response = GitalyClient.call(@storage, :remote_service, + :fetch_internal_remote, request, + remote_storage: repository.storage) + + response.result + end + + def update_remote_mirror(ref_name, only_branches_matching) + req_enum = Enumerator.new do |y| + y.yield Gitaly::UpdateRemoteMirrorRequest.new( + repository: @gitaly_repo, + ref_name: ref_name + ) + + current_size = 0 + + slices = only_branches_matching.slice_before do |branch_name| + current_size += branch_name.bytesize + + next false if current_size < MAX_MSG_SIZE + + current_size = 0 + end + + slices.each do |slice| + y.yield Gitaly::UpdateRemoteMirrorRequest.new(only_branches_matching: slice) + end + end + + GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index c1f95396878..60706b4f0d8 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -1,6 +1,10 @@ module Gitlab module GitalyClient class RepositoryService + include Gitlab::EncodingHelper + + MAX_MSG_SIZE = 128.kilobytes.freeze + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository @@ -41,8 +45,11 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end - def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false) - request = Gitaly::FetchRemoteRequest.new(repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags) + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:) + request = Gitaly::FetchRemoteRequest.new( + repository: @gitaly_repo, remote: remote, force: forced, + no_tags: no_tags, timeout: timeout + ) if ssh_auth&.ssh_import? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? @@ -72,13 +79,61 @@ module Gitlab def find_merge_base(*revisions) request = Gitaly::FindMergeBaseRequest.new( repository: @gitaly_repo, - revisions: revisions.map { |r| GitalyClient.encode(r) } + revisions: revisions.map { |r| encode_binary(r) } ) response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request) response.base.presence end + def fork_repository(source_repository) + request = Gitaly::CreateForkRequest.new( + repository: @gitaly_repo, + source_repository: source_repository.gitaly_repository + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_fork, + request, + remote_storage: source_repository.storage, + timeout: GitalyClient.default_timeout + ) + end + + def import_repository(source) + request = Gitaly::CreateRepositoryFromURLRequest.new( + repository: @gitaly_repo, + url: source + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_repository_from_url, + request, + timeout: GitalyClient.default_timeout + ) + end + + def rebase_in_progress?(rebase_id) + request = Gitaly::IsRebaseInProgressRequest.new( + repository: @gitaly_repo, + rebase_id: rebase_id.to_s + ) + + response = GitalyClient.call( + @storage, + :repository_service, + :is_rebase_in_progress, + request, + timeout: GitalyClient.default_timeout + ) + + response.in_progress + end + def fetch_source_branch(source_repository, source_branch, local_ref) request = Gitaly::FetchSourceBranchRequest.new( repository: @gitaly_repo, @@ -108,6 +163,75 @@ module Gitlab return response.error.b, 1 end end + + def create_bundle(save_path) + request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + :create_bundle, + request, + timeout: GitalyClient.default_timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + end + + def create_from_bundle(bundle_path) + request = Gitaly::CreateRepositoryFromBundleRequest.new(repository: @gitaly_repo) + enum = Enumerator.new do |y| + File.open(bundle_path, 'rb') do |f| + while data = f.read(MAX_MSG_SIZE) + request.data = data + + y.yield request + + request = Gitaly::CreateRepositoryFromBundleRequest.new + end + end + end + + GitalyClient.call( + @storage, + :repository_service, + :create_repository_from_bundle, + enum, + timeout: GitalyClient.default_timeout + ) + end + + def write_ref(ref_path, ref, old_ref, shell) + request = Gitaly::WriteRefRequest.new( + repository: @gitaly_repo, + ref: ref_path.b, + revision: ref.b, + shell: shell + ) + request.old_revision = old_ref.b unless old_ref.nil? + + response = GitalyClient.call(@storage, :repository_service, :write_ref, request) + + raise Gitlab::Git::CommandError, encode!(response.error) if response.error.present? + + true + end + + def write_config(full_path:) + request = Gitaly::WriteConfigRequest.new(repository: @gitaly_repo, full_path: full_path) + response = GitalyClient.call( + @storage, + :repository_service, + :write_config, + request, + timeout: GitalyClient.fast_timeout + ) + + raise Gitlab::Git::OSError.new(response.error) unless response.error.empty? + end end end end diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb new file mode 100644 index 00000000000..2e1076d1f66 --- /dev/null +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -0,0 +1,16 @@ +module Gitlab + module GitalyClient + # Meant for extraction of server data, and later maybe to perform misc task + # + # Not meant for connection logic, look in Gitlab::GitalyClient + class ServerService + def initialize(storage) + @storage = storage + end + + def info + GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index b1a033280b4..a8c6d478de8 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -12,12 +12,18 @@ module Gitlab Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, - gl_repository: gl_repository, + gl_repository: gl_repository.to_s, git_object_directory: git_object_directory.to_s, git_alternate_object_directories: git_alternate_object_directories ) end + def git_repository(gitaly_repository) + Gitlab::Git::Repository.new(gitaly_repository.storage_name, + gitaly_repository.relative_path, + gitaly_repository.gl_repository) + end + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) if gitaly_tag.target_commit.present? commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 7339468e911..a02d15db5dd 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -4,6 +4,7 @@ module Gitlab ATTRS = %i(title format url_path path name historical raw_data).freeze include AttributesBag + include Gitlab::EncodingHelper def initialize(params) super @@ -11,6 +12,10 @@ module Gitlab # All gRPC strings in a response are frozen, so we get an unfrozen # version here so appending to `raw_data` doesn't blow up. @raw_data = @raw_data.dup + + @title = encode_utf8(@title) + @path = encode_utf8(@path) + @name = encode_utf8(@name) end def historical? diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 337d225d081..8e87a8cc36f 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -3,6 +3,8 @@ require 'stringio' module Gitlab module GitalyClient class WikiService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def initialize(repository) @@ -13,12 +15,12 @@ module Gitlab def write_page(name, format, content, commit_details) request = Gitaly::WikiWritePageRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), + name: encode_binary(name), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -39,13 +41,13 @@ module Gitlab def update_page(page_path, title, format, content, commit_details) request = Gitaly::WikiUpdatePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), - title: GitalyClient.encode(title), + page_path: encode_binary(page_path), + title: encode_binary(title), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -63,7 +65,7 @@ module Gitlab def delete_page(page_path, commit_details) request = Gitaly::WikiDeletePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), + page_path: encode_binary(page_path), commit_details: gitaly_commit_details(commit_details) ) @@ -73,9 +75,9 @@ module Gitlab def find_page(title:, version: nil, dir: nil) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, - title: GitalyClient.encode(title), - revision: GitalyClient.encode(version), - directory: GitalyClient.encode(dir) + title: encode_binary(title), + revision: encode_binary(version), + directory: encode_binary(dir) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) @@ -102,8 +104,8 @@ module Gitlab def find_file(name, revision) request = Gitaly::WikiFindFileRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), - revision: GitalyClient.encode(revision) + name: encode_binary(name), + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request) @@ -125,6 +127,18 @@ module Gitlab wiki_file end + def get_formatted_data(title:, dir: nil, version: nil) + request = Gitaly::WikiGetFormattedDataRequest.new( + repository: @gitaly_repo, + title: encode_binary(title), + revision: encode_binary(version), + directory: encode_binary(dir) + ) + + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request) + response.reduce("") { |memo, msg| memo << msg.data } + end + private # If a block is given and the yielded value is true, iteration will be @@ -158,9 +172,9 @@ module Gitlab def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( - name: GitalyClient.encode(commit_details.name), - email: GitalyClient.encode(commit_details.email), - message: GitalyClient.encode(commit_details.message) + name: encode_binary(commit_details.name), + email: encode_binary(commit_details.email), + message: encode_binary(commit_details.message) ) end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 5da9befa08e..4f160e4a447 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -14,6 +14,8 @@ module Gitlab # puts label.name # end class Client + include ::Gitlab::Utils::StrongMemoize + attr_reader :octokit # A single page of data and the corresponding page number. @@ -173,7 +175,9 @@ module Gitlab end def rate_limiting_enabled? - @rate_limiting_enabled ||= api_endpoint.include?('.github.com') + strong_memoize(:rate_limiting_enabled) do + api_endpoint.include?('.github.com') + end end def api_endpoint diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 5437e32e9f1..e70361c163b 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -57,10 +57,7 @@ module Gitlab end def commit_exists?(sha) - project.repository.lookup(sha) - true - rescue Rugged::Error - false + project.repository.commit(sha).present? end def collection_method diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index bb7439a0641..be1334ca98a 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -13,7 +13,7 @@ module Gitlab :diff_hunk, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_ID_REGEX = /\/pull\/(?<iid>\d+)/i + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i # Builds a diff note from a GitHub API response. # diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index a68bc4c002f..070e3b2db8d 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_TYPE_REGEX = /\/(?<type>(pull|issues))\/(?<iid>\d+)/i + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 1fe076cf909..ba04387022d 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,22 +3,22 @@ module Gitlab module GonHelper include WebpackHelper - include Gitlab::CurrentSettings def add_gon_variables gon.api_version = 'v4' gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.max_file_size = current_application_settings.max_attachment_size + gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path + gon.sprite_file_icons = IconsHelper.sprite_file_icons_path if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index ab38c0c3e34..46b49128140 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -302,6 +302,7 @@ module Gitlab else "#{project.namespace.full_path}/#{name}##{id}" end + text = "~~#{text}~~" if deleted text end @@ -329,6 +330,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? @@ -352,6 +354,7 @@ module Gitlab if content.blank? content = "*(No description has been entered for this issue)*" end + body << content if attachments.any? diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 0f4ba6f83fc..672b5579dfd 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -4,12 +4,8 @@ module Gitlab def initialize(commit) @commit = commit - @signature_text, @signed_text = - begin - Rugged::Commit.extract_signature(@commit.project.repository.rugged, @commit.sha) - rescue Rugged::OdbError - nil - end + repo = commit.project.repository.raw_repository + @signature_text, @signed_text = Gitlab::Git::Commit.extract_signature(repo, commit.sha) end def has_signature? diff --git a/lib/gitlab/grape_logging/loggers/user_logger.rb b/lib/gitlab/grape_logging/loggers/user_logger.rb new file mode 100644 index 00000000000..fa172861967 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/user_logger.rb @@ -0,0 +1,18 @@ +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log the user who performed the Grape API action by retrieving +# the user context from the request environment. +module Gitlab + module GrapeLogging + module Loggers + class UserLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + params = request.env[::API::Helpers::API_USER_ENV] + + return {} unless params + + params.slice(:user_id, :username) + end + end + end + end +end diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb new file mode 100644 index 00000000000..11416c002e3 --- /dev/null +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -0,0 +1,53 @@ +module Gitlab + module HealthChecks + class GitalyCheck + extend BaseAbstractCheck + + METRIC_PREFIX = 'gitaly_health_check'.freeze + + class << self + def readiness + repository_storages.map do |storage_name| + check(storage_name) + end + end + + def metrics + repository_storages.flat_map do |storage_name| + result, elapsed = with_timing { check(storage_name) } + labels = { shard: storage_name } + + [ + metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels), + metric("#{metric_prefix}_latency_seconds", elapsed, **labels) + ].flatten + end + end + + def check(storage_name) + serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name) + result = serv.check + HealthChecks::Result.new(result[:success], result[:message], shard: storage_name) + end + + private + + def metric_prefix + METRIC_PREFIX + end + + def successful?(result) + result[:success] + end + + def repository_storages + storages.keys + end + + def storages + Gitlab.config.repositories.storages + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index e29dd0d5b0e..f9b1a3caf5e 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -7,7 +7,6 @@ module Gitlab closed_at confidential created_at - deleted_at description due_date id diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index ae9b68eb648..aff786864f2 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -5,7 +5,6 @@ module Gitlab assignee_id author_id created_at - deleted_at description head_pipeline_id id diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 2066005dddc..af203ff711d 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.1'.freeze + VERSION = '0.2.2'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 0135b3c6f22..2f163db936b 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -11,10 +11,6 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end - def git_bundle(repo_path:, bundle_path:) - execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) - end - def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_MODE) FileUtils.chmod(DEFAULT_MODE, path) diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 989342389bc..0f4c3498036 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -17,12 +17,16 @@ module Gitlab def import mkdir_p(@shared.export_path) + remove_symlinks! + wait_for_archived_file do decompress_archive end rescue => e @shared.error(e) false + ensure + remove_symlinks! end private @@ -43,7 +47,7 @@ module Gitlab raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result - remove_symlinks! + result end def remove_symlinks! @@ -55,7 +59,7 @@ module Gitlab end def extracted_files - Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ /.*\/\.{1,2}$/ } + Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ %r{.*/\.{1,2}$} } end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f2b193c79cb..2daed10f678 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -49,8 +49,8 @@ project_tree: - :author - events: - :push_event_payload - - :stages - - :statuses + - stages: + - :statuses - :auto_devops - :triggers - :pipeline_schedules diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c518943be59..4b5f9f3a926 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -148,6 +148,7 @@ module Gitlab else relation_hash = relation_item[sub_relation.to_s] end + [relation_hash, sub_relation] end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d7d1b05e8b9..759833a5ee5 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -62,6 +62,7 @@ module Gitlab when :notes then setup_note when :project_label, :project_labels then setup_label when :milestone, :milestones then setup_milestone + when 'Ci::Pipeline' then setup_pipeline else @relation_hash['project_id'] = @project.id end @@ -112,9 +113,7 @@ module Gitlab @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') - imported_object do |object| - object.commit_id = nil - end + imported_object elsif @relation_name == :merge_requests MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse! else @@ -140,13 +139,12 @@ module Gitlab end def setup_label - return unless @relation_hash['type'] == 'GroupLabel' - # If there's no group, move the label to a project label - if @relation_hash['group_id'] + if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] @relation_hash['project_id'] = nil @relation_name = :group_label else + @relation_hash['group_id'] = nil @relation_hash['type'] = 'ProjectLabel' end end @@ -182,8 +180,9 @@ module Gitlab end def imported_object - yield(existing_or_new_object) if block_given? - existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing) + if existing_or_new_object.respond_to?(:importing) + existing_or_new_object.importing = true + end existing_or_new_object rescue ActiveRecord::RecordNotUnique @@ -211,6 +210,14 @@ module Gitlab @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end + def setup_pipeline + @relation_hash.fetch('stages').each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. @@ -259,6 +266,7 @@ module Gitlab else %w[title group_id] end + finder_hash = parsed_relation_hash.slice(*finder_attributes) if label? diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 32ca2809b2f..5a9bbceac67 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -13,7 +13,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - gitlab_shell.import_repository(@project.repository_storage_path, @project.disk_path, @path_to_bundle) + @project.repository.create_from_bundle(@path_to_bundle) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index a7028a32570..695462c7dd2 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -21,7 +21,7 @@ module Gitlab def bundle_to_disk mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + @project.repository.bundle_to_disk(@full_path) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 6130c124dd1..2daeba90a51 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -37,7 +37,7 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.export_path, '..', Gitlab::ImportExport.export_filename(project: @project)) + @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 9fd0b709ef2..b34cafc6876 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -9,18 +9,35 @@ module Gitlab end def export_path - @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) + @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) + end + + def archive_path + @archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path) end def error(error) error_out(error.message, caller[0].dup) @errors << error.message + # Debug: - Rails.logger.error(error.backtrace.join("\n")) + if error.backtrace + Rails.logger.error("Import/Export backtrace: #{error.backtrace.join("\n")}") + else + Rails.logger.error("No backtrace found") + end end private + def relative_path + File.join(opts[:relative_path], SecureRandom.hex) + end + + def relative_archive_path + File.join(opts[:relative_path], '..') + end + def error_out(message, caller) Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 627a487d577..2f08dda55fd 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -17,15 +17,13 @@ module Gitlab false end - private + def uploads_path + FileUploader.absolute_base_dir(@project) + end def uploads_export_path File.join(@shared.export_path, 'uploads') end - - def uploads_path - FileUploader.dynamic_path_segment(@project) - end end end end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 1e6722a7bba..5fa2e101e29 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -10,7 +10,7 @@ module Gitlab def bundle_to_disk(full_path) mkdir_p(@shared.export_path) - git_bundle(repo_path: path_to_repo, bundle_path: full_path) + @wiki.repository.bundle_to_disk(full_path) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index eeb03625479..60d5fa4d29a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,6 +7,7 @@ module Gitlab module ImportSources ImportSource = Struct.new(:name, :title, :importer) + # We exclude `bare_repository` here as it has no import class associated ImportTable = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb new file mode 100644 index 00000000000..f85b6e9197f --- /dev/null +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -0,0 +1,23 @@ +module Gitlab + # + # Calculates the fingerprint of a given key without using + # openssh key validations. For this reason, only use + # for calculating the fingerprint to find the key with it. + # + # DO NOT use it for checking the validity of a ssh key. + # + class InsecureKeyFingerprint + attr_accessor :key + + # + # Gets the base64 encoded string representing a rsa or dsa key + # + def initialize(key_base64) + @key = key_base64 + end + + def fingerprint + OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':') + end + end +end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 407cdefc04d..0f0588b8b23 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -1,96 +1,8 @@ module Gitlab module Kubernetes - class Helm + module Helm HELM_VERSION = '2.7.0'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze - INSTALL_DEPS = <<-EOS.freeze - set -eo pipefail - apk add -U ca-certificates openssl >/dev/null - wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null - mv /tmp/linux-amd64/helm /usr/bin/ - EOS - - InstallCommand = Struct.new(:name, :install_helm, :chart) do - def pod_name - "install-#{name}" - end - end - - def initialize(kubeclient) - @kubeclient = kubeclient - @namespace = Gitlab::Kubernetes::Namespace.new(NAMESPACE, kubeclient) - end - - def install(command) - @namespace.ensure_exists! - @kubeclient.create_pod(pod_resource(command)) - end - - ## - # Returns Pod phase - # - # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase - # - # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" - # - def installation_status(pod_name) - @kubeclient.get_pod(pod_name, @namespace.name).status.phase - end - - def installation_log(pod_name) - @kubeclient.get_pod_log(pod_name, @namespace.name).body - end - - def delete_installation_pod!(pod_name) - @kubeclient.delete_pod(pod_name, @namespace.name) - end - - private - - def pod_resource(command) - labels = { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name } - metadata = { name: command.pod_name, namespace: @namespace.name, labels: labels } - container = { - name: 'helm', - image: 'alpine:3.6', - env: generate_pod_env(command), - command: %w(/bin/sh), - args: %w(-c $(COMMAND_SCRIPT)) - } - spec = { containers: [container], restartPolicy: 'Never' } - - ::Kubeclient::Resource.new(metadata: metadata, spec: spec) - end - - def generate_pod_env(command) - { - HELM_VERSION: HELM_VERSION, - TILLER_NAMESPACE: @namespace.name, - COMMAND_SCRIPT: generate_script(command) - }.map { |key, value| { name: key, value: value } } - end - - def generate_script(command) - [ - INSTALL_DEPS, - helm_init_command(command), - helm_install_command(command) - ].join("\n") - end - - def helm_init_command(command) - if command.install_helm - 'helm init >/dev/null' - else - 'helm init --client-only >/dev/null' - end - end - - def helm_install_command(command) - return if command.chart.nil? - - "helm install #{command.chart} --name #{command.name} --namespace #{@namespace.name} >/dev/null" - end end end end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb new file mode 100644 index 00000000000..737081ddc5b --- /dev/null +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -0,0 +1,42 @@ +module Gitlab + module Kubernetes + module Helm + class Api + def initialize(kubeclient) + @kubeclient = kubeclient + @namespace = Gitlab::Kubernetes::Namespace.new(Gitlab::Kubernetes::Helm::NAMESPACE, kubeclient) + end + + def install(command) + @namespace.ensure_exists! + @kubeclient.create_pod(pod_resource(command)) + end + + ## + # Returns Pod phase + # + # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + # + # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" + # + def installation_status(pod_name) + @kubeclient.get_pod(pod_name, @namespace.name).status.phase + end + + def installation_log(pod_name) + @kubeclient.get_pod_log(pod_name, @namespace.name).body + end + + def delete_installation_pod!(pod_name) + @kubeclient.delete_pod(pod_name, @namespace.name) + end + + private + + def pod_resource(command) + Gitlab::Kubernetes::Helm::Pod.new(command, @namespace.name, @kubeclient).generate + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb new file mode 100644 index 00000000000..bf6981035f4 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -0,0 +1,57 @@ +module Gitlab + module Kubernetes + module Helm + class InstallCommand + attr_reader :name, :install_helm, :chart, :chart_values_file + + def initialize(name, install_helm: false, chart: false, chart_values_file: false) + @name = name + @install_helm = install_helm + @chart = chart + @chart_values_file = chart_values_file + end + + def pod_name + "install-#{name}" + end + + def generate_script(namespace_name) + [ + install_dps_command, + init_command, + complete_command(namespace_name) + ].join("\n") + end + + private + + def init_command + if install_helm + 'helm init >/dev/null' + else + 'helm init --client-only >/dev/null' + end + end + + def complete_command(namespace_name) + return unless chart + + if chart_values_file + "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null" + else + "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null" + end + end + + def install_dps_command + <<~HEREDOC + set -eo pipefail + apk add -U ca-certificates openssl >/dev/null + wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null + mv /tmp/linux-amd64/helm /usr/bin/ + HEREDOC + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb new file mode 100644 index 00000000000..ca5e06009fa --- /dev/null +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -0,0 +1,95 @@ +module Gitlab + module Kubernetes + module Helm + class Pod + def initialize(command, namespace_name, kubeclient) + @command = command + @namespace_name = namespace_name + @kubeclient = kubeclient + end + + def generate + spec = { containers: [container_specification], restartPolicy: 'Never' } + + if command.chart_values_file + create_config_map + spec[:volumes] = volumes_specification + end + + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) + end + + private + + attr_reader :command, :namespace_name, :kubeclient + + def container_specification + container = { + name: 'helm', + image: 'alpine:3.6', + env: generate_pod_env(command), + command: %w(/bin/sh), + args: %w(-c $(COMMAND_SCRIPT)) + } + container[:volumeMounts] = volume_mounts_specification if command.chart_values_file + container + end + + def labels + { + 'gitlab.org/action': 'install', + 'gitlab.org/application': command.name + } + end + + def metadata + { + name: command.pod_name, + namespace: namespace_name, + labels: labels + } + end + + def volume_mounts_specification + [ + { + name: 'configuration-volume', + mountPath: "/data/helm/#{command.name}/config" + } + ] + end + + def volumes_specification + [ + { + name: 'configuration-volume', + configMap: { + name: "values-content-configuration-#{command.name}", + items: [{ key: 'values', path: 'values.yaml' }] + } + } + ] + end + + def generate_pod_env(command) + { + HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, + TILLER_NAMESPACE: namespace_name, + COMMAND_SCRIPT: command.generate_script(namespace_name) + }.map { |key, value| { name: key, value: value } } + end + + def create_config_map + resource = ::Kubeclient::Resource.new + resource.metadata = { + name: "values-content-configuration-#{command.name}", + namespace: namespace_name, + labels: { name: "values-content-configuration-#{command.name}" } + } + resource.data = { values: File.read(command.chart_values_file) } + kubeclient.create_config_map(resource) + end + end + end + end +end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 0afaa2306b5..76863e77dc3 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -74,7 +74,7 @@ module Gitlab def user_options(fields, value, limit) options = { - attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + attributes: Gitlab::LDAP::Person.ldap_attributes(config), base: config.base } diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index c8f19cd52d5..cde60addcf7 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -42,6 +42,7 @@ module Gitlab else self.class.invalid_provider(provider) end + @options = config_for(@provider) # Use @provider, not provider end @@ -148,7 +149,7 @@ module Gitlab def default_attributes { - 'username' => %w(uid userid sAMAccountName), + 'username' => %w(uid sAMAccountName userid), 'email' => %w(mail email userPrincipalName), 'name' => 'cn', 'first_name' => 'givenName', diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 38d7a9ba2f5..e81cec6ba1a 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -6,6 +6,8 @@ module Gitlab # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") + InvalidEntryError = Class.new(StandardError) + attr_accessor :entry, :provider def self.find_by_uid(uid, adapter) @@ -29,11 +31,12 @@ module Gitlab def self.ldap_attributes(config) [ - 'dn', # Used in `dn` - config.uid, # Used in `uid` - *config.attributes['name'], # Used in `name` - *config.attributes['email'] # Used in `email` - ] + 'dn', + config.uid, + *config.attributes['name'], + *config.attributes['email'], + *config.attributes['username'] + ].compact.uniq end def self.normalize_dn(dn) @@ -60,6 +63,8 @@ module Gitlab Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @provider = provider + + validate_entry end def name @@ -71,7 +76,13 @@ module Gitlab end def username - uid + username = attribute_value(:username) + + # Depending on the attribute, multiple values may + # be returned. We need only one for username. + # Ex. `uid` returns only one value but `mail` may + # return an array of multiple email addresses. + [username].flatten.first end def email @@ -104,6 +115,19 @@ module Gitlab entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end + + def validate_entry + allowed_attrs = self.class.ldap_attributes(config).map(&:downcase) + + # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare. + entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase } + invalid_attrs = entry_attrs - allowed_attrs + + if invalid_attrs.any? + raise InvalidEntryError, + "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}" + end + end end end end diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index 41e7eac4d08..cbabe5454ca 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,8 +1,6 @@ module Gitlab module LegacyGithubImport class ProjectCreator - include Gitlab::CurrentSettings - attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -36,7 +34,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility end # diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 4779755bb22..7d63ca5627d 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,7 +1,7 @@ module Gitlab module Metrics - extend Gitlab::Metrics::InfluxDb - extend Gitlab::Metrics::Prometheus + include Gitlab::Metrics::InfluxDb + include Gitlab::Metrics::Prometheus def self.enabled? influx_metrics_enabled? || prometheus_metrics_enabled? diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 877cebf6786..66f30e3b397 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -1,178 +1,187 @@ module Gitlab module Metrics module InfluxDb - include Gitlab::CurrentSettings - extend self + extend ActiveSupport::Concern + include Gitlab::Metrics::Methods + + EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze MUTEX = Mutex.new private_constant :MUTEX - def influx_metrics_enabled? - settings[:enabled] || false - end + class_methods do + def influx_metrics_enabled? + settings[:enabled] || false + end - # Prometheus histogram buckets used for arbitrary code measurements - EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1].freeze - RAILS_ROOT = Rails.root.to_s - METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s - PATH_REGEX = /^#{RAILS_ROOT}\/?/ - - def settings - @settings ||= { - enabled: current_application_settings[:metrics_enabled], - pool_size: current_application_settings[:metrics_pool_size], - timeout: current_application_settings[:metrics_timeout], - method_call_threshold: current_application_settings[:metrics_method_call_threshold], - host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15, - packet_size: current_application_settings[:metrics_packet_size] || 1 - } - end + # Prometheus histogram buckets used for arbitrary code measurements + + def settings + @settings ||= begin + current_settings = Gitlab::CurrentSettings.current_application_settings + + { + enabled: current_settings[:metrics_enabled], + pool_size: current_settings[:metrics_pool_size], + timeout: current_settings[:metrics_timeout], + method_call_threshold: current_settings[:metrics_method_call_threshold], + host: current_settings[:metrics_host], + port: current_settings[:metrics_port], + sample_interval: current_settings[:metrics_sample_interval] || 15, + packet_size: current_settings[:metrics_packet_size] || 1 + } + end + end - def mri? - RUBY_ENGINE == 'ruby' - end + def mri? + RUBY_ENGINE == 'ruby' + end - def method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - # in milliseconds - @method_call_threshold ||= settings[:method_call_threshold] - end + def method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + # in milliseconds + @method_call_threshold ||= settings[:method_call_threshold] + end - def submit_metrics(metrics) - prepared = prepare_metrics(metrics) + def submit_metrics(metrics) + prepared = prepare_metrics(metrics) - pool&.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - begin - connection.write_points(slice) - rescue StandardError + pool&.with do |connection| + prepared.each_slice(settings[:packet_size]) do |slice| + begin + connection.write_points(slice) + rescue StandardError + end end end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) end - rescue Errno::EADDRNOTAVAIL, SocketError => ex - Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') - Gitlab::EnvironmentLogger.error(ex) - end - def prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys + def prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end end + + new_hash end + end - new_hash + def escape_value(value) + value.to_s.gsub('=', '\\=') end - end - def escape_value(value) - value.to_s.gsub('=', '\\=') - end + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) + cpu_time = cpu_stop - cpu_start + + real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do + docstring "Measure #{name}" + base_labels Transaction::BASE_LABELS + buckets EXECUTION_MEASUREMENT_BUCKETS + end - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # User.find_by_username(some_username) - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) - cpu_time = cpu_stop - cpu_start - - Gitlab::Metrics.histogram("gitlab_#{name}_real_duration_seconds".to_sym, - "Measure #{name}", - Transaction::BASE_LABELS, - EXECUTION_MEASUREMENT_BUCKETS) - .observe(trans.labels, real_time) - - Gitlab::Metrics.histogram("gitlab_#{name}_cpu_duration_seconds".to_sym, - "Measure #{name}", - Transaction::BASE_LABELS, - EXECUTION_MEASUREMENT_BUCKETS) - .observe(trans.labels, cpu_time / 1000.0) - - # InfluxDB stores the _real_time time values as milliseconds - trans.increment("#{name}_real_time", real_time * 1000, false) - trans.increment("#{name}_cpu_time", cpu_time, false) - trans.increment("#{name}_call_count", 1, false) - - retval - end + real_duration_seconds.observe(trans.labels, real_time) - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def action=(action) - trans = current_transaction + cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do + docstring "Measure #{name}" + base_labels Transaction::BASE_LABELS + buckets EXECUTION_MEASUREMENT_BUCKETS + with_feature "prometheus_metrics_measure_#{name}_cpu_duration" + end + cpu_duration_seconds.observe(trans.labels, cpu_time) - trans&.action = action - end + # InfluxDB stores the _real_time and _cpu_time time values as milliseconds + trans.increment("#{name}_real_time", real_time.in_milliseconds, false) + trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false) + trans.increment("#{name}_call_count", 1, false) - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def add_event(*args) - trans = current_transaction + retval + end - trans&.add_event(*args) - end + # Sets the action of the current transaction (if any) + # + # action - The name of the action. + def action=(action) + trans = current_transaction - # Returns the prefix to use for the name of a series. - def series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' - end + trans&.action = action + end - # Allow access from other metrics related middlewares - def current_transaction - Transaction.current - end + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def add_event(*args) + trans = current_transaction + + trans&.add_event(*args) + end + + # Returns the prefix to use for the name of a series. + def series_prefix + @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + end + + # Allow access from other metrics related middlewares + def current_transaction + Transaction.current + end - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - # rubocop:disable Gitlab/ModuleWithInstanceVariables - def pool - if influx_metrics_enabled? - if @pool.nil? - MUTEX.synchronize do - @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client - .new(udp: { host: host, port: port }) + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def pool + if influx_metrics_enabled? + if @pool.nil? + MUTEX.synchronize do + @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] + + InfluxDB::Client + .new(udp: { host: host, port: port }) + end end end + + @pool end - @pool end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end - # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 329b07af5db..b11520a79bb 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -4,26 +4,15 @@ module Gitlab module Metrics # Class for tracking timing information about method calls class MethodCall - @@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false) - @@measurement_enabled_cache_expires_at = Concurrent::AtomicFixnum.new(Time.now.to_i) - MUTEX = Mutex.new + include Gitlab::Metrics::Methods BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels - def self.call_duration_histogram - return @call_duration_histogram if @call_duration_histogram - - MUTEX.synchronize do - @call_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_duration_seconds, - 'Method calls real duration', - Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.01, 0.05, 0.1, 0.5, 1]) - end - end - - def self.measurement_enabled_cache_expires_at - @@measurement_enabled_cache_expires_at + define_histogram :gitlab_method_call_duration_seconds do + docstring 'Method calls real duration' + base_labels Transaction::BASE_LABELS.merge(BASE_LABELS) + buckets [0.01, 0.05, 0.1, 0.5, 1] + with_feature :prometheus_metrics_method_instrumentation end # name - The full name of the method (including namespace) such as @@ -53,8 +42,8 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - if call_measurement_enabled? && above_threshold? - self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time) + if above_threshold? + self.class.gitlab_method_call_duration_seconds.observe(@transaction.labels.merge(labels), real_time) end retval @@ -78,17 +67,6 @@ module Gitlab def above_threshold? real_time.in_milliseconds >= Metrics.method_call_threshold end - - def call_measurement_enabled? - expires_at = @@measurement_enabled_cache_expires_at.value - if expires_at < Time.now.to_i - if @@measurement_enabled_cache_expires_at.compare_and_set(expires_at, 1.minute.from_now.to_i) - @@measurement_enabled_cache.value = Feature.get(:prometheus_metrics_method_instrumentation).enabled? - end - end - - @@measurement_enabled_cache.value - end end end end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb new file mode 100644 index 00000000000..cd7c1e507f7 --- /dev/null +++ b/lib/gitlab/metrics/methods.rb @@ -0,0 +1,129 @@ +# rubocop:disable Style/ClassVars + +module Gitlab + module Metrics + module Methods + extend ActiveSupport::Concern + + included do + @@_metric_provider_mutex ||= Mutex.new + @@_metrics_provider_cache = {} + end + + class_methods do + def reload_metric!(name) + @@_metrics_provider_cache.delete(name) + end + + private + + def define_metric(type, name, opts = {}, &block) + if respond_to?(name) + raise ArgumentError, "method #{name} already exists" + end + + define_singleton_method(name) do + # inlining fetch_metric method to avoid method call overhead when instrumenting hot spots + @@_metrics_provider_cache[name] || init_metric(type, name, opts, &block) + end + end + + def fetch_metric(type, name, opts = {}, &block) + @@_metrics_provider_cache[name] || init_metric(type, name, opts, &block) + end + + def init_metric(type, name, opts = {}, &block) + options = MetricOptions.new(opts) + options.evaluate(&block) + + if disabled_by_feature(options) + synchronized_cache_fill(name) { NullMetric.instance } + else + synchronized_cache_fill(name) { build_metric!(type, name, options) } + end + end + + def synchronized_cache_fill(key) + @@_metric_provider_mutex.synchronize do + @@_metrics_provider_cache[key] ||= yield + end + end + + def disabled_by_feature(options) + options.with_feature && !Feature.get(options.with_feature).enabled? + end + + def build_metric!(type, name, options) + case type + when :gauge + Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) + when :counter + Gitlab::Metrics.counter(name, options.docstring, options.base_labels) + when :histogram + Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) + when :summary + raise NotImplementedError, "summary metrics are not currently supported" + else + raise ArgumentError, "uknown metric type #{type}" + end + end + + # Fetch and/or initialize counter metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_counter(name, opts = {}, &block) + fetch_metric(:counter, name, opts, &block) + end + + # Fetch and/or initialize gauge metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_gauge(name, opts = {}, &block) + fetch_metric(:gauge, name, opts, &block) + end + + # Fetch and/or initialize histogram metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_histogram(name, opts = {}, &block) + fetch_metric(:histogram, name, opts, &block) + end + + # Fetch and/or initialize summary metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_summary(name, opts = {}, &block) + fetch_metric(:summary, name, opts, &block) + end + + # Define metric accessor method for a Counter + # @param [Symbol] name + # @param [Hash] opts + def define_counter(name, opts = {}, &block) + define_metric(:counter, name, opts, &block) + end + + # Define metric accessor method for a Gauge + # @param [Symbol] name + # @param [Hash] opts + def define_gauge(name, opts = {}, &block) + define_metric(:gauge, name, opts, &block) + end + + # Define metric accessor method for a Histogram + # @param [Symbol] name + # @param [Hash] opts + def define_histogram(name, opts = {}, &block) + define_metric(:histogram, name, opts, &block) + end + + # Define metric accessor method for a Summary + # @param [Symbol] name + # @param [Hash] opts + def define_summary(name, opts = {}, &block) + define_metric(:summary, name, opts, &block) + end + end + end + end +end diff --git a/lib/gitlab/metrics/methods/metric_options.rb b/lib/gitlab/metrics/methods/metric_options.rb new file mode 100644 index 00000000000..70e122d4e15 --- /dev/null +++ b/lib/gitlab/metrics/methods/metric_options.rb @@ -0,0 +1,61 @@ +module Gitlab + module Metrics + module Methods + class MetricOptions + SMALL_NETWORK_BUCKETS = [0.005, 0.01, 0.1, 1, 10].freeze + + def initialize(options = {}) + @multiprocess_mode = options[:multiprocess_mode] || :all + @buckets = options[:buckets] || SMALL_NETWORK_BUCKETS + @base_labels = options[:base_labels] || {} + @docstring = options[:docstring] + @with_feature = options[:with_feature] + end + + # Documentation describing metric in metrics endpoint '/-/metrics' + def docstring(docstring = nil) + @docstring = docstring unless docstring.nil? + + @docstring + end + + # Gauge aggregation mode for multiprocess metrics + # - :all (default) returns each gauge for every process + # - :livesum all process'es gauges summed up + # - :max maximum value of per process gauges + # - :min minimum value of per process gauges + def multiprocess_mode(mode = nil) + @multiprocess_mode = mode unless mode.nil? + + @multiprocess_mode + end + + # Measurement buckets for histograms + def buckets(buckets = nil) + @buckets = buckets unless buckets.nil? + + @buckets + end + + # Base labels are merged with per metric labels + def base_labels(base_labels = nil) + @base_labels = base_labels unless base_labels.nil? + + @base_labels + end + + # Use feature toggle to control whether certain metric is enabled/disabled + def with_feature(name = nil) + @with_feature = name unless name.nil? + + @with_feature + end + + def evaluate(&block) + instance_eval(&block) if block_given? + self + end + end + end + end +end diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb index 3b5a2907195..aabada5c21a 100644 --- a/lib/gitlab/metrics/null_metric.rb +++ b/lib/gitlab/metrics/null_metric.rb @@ -2,6 +2,8 @@ module Gitlab module Metrics # Mocks ::Prometheus::Client::Metric and all derived metrics class NullMetric + include Singleton + def method_missing(name, *args, &block) nil end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index b0b8e8436db..d12ba0ec176 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -3,73 +3,76 @@ require 'prometheus/client' module Gitlab module Metrics module Prometheus - include Gitlab::CurrentSettings - include Gitlab::Utils::StrongMemoize + extend ActiveSupport::Concern REGISTRY_MUTEX = Mutex.new PROVIDER_MUTEX = Mutex.new - def metrics_folder_present? - multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir + class_methods do + include Gitlab::Utils::StrongMemoize - multiprocess_files_dir && - ::Dir.exist?(multiprocess_files_dir) && - ::File.writable?(multiprocess_files_dir) - end + def metrics_folder_present? + multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir - def prometheus_metrics_enabled? - strong_memoize(:prometheus_metrics_enabled) do - prometheus_metrics_enabled_unmemoized + multiprocess_files_dir && + ::Dir.exist?(multiprocess_files_dir) && + ::File.writable?(multiprocess_files_dir) + end + + def prometheus_metrics_enabled? + strong_memoize(:prometheus_metrics_enabled) do + prometheus_metrics_enabled_unmemoized + end end - end - def registry - strong_memoize(:registry) do - REGISTRY_MUTEX.synchronize do - strong_memoize(:registry) do - ::Prometheus::Client.registry + def registry + strong_memoize(:registry) do + REGISTRY_MUTEX.synchronize do + strong_memoize(:registry) do + ::Prometheus::Client.registry + end end end end - end - def counter(name, docstring, base_labels = {}) - safe_provide_metric(:counter, name, docstring, base_labels) - end + def counter(name, docstring, base_labels = {}) + safe_provide_metric(:counter, name, docstring, base_labels) + end - def summary(name, docstring, base_labels = {}) - safe_provide_metric(:summary, name, docstring, base_labels) - end + def summary(name, docstring, base_labels = {}) + safe_provide_metric(:summary, name, docstring, base_labels) + end - def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) - safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode) - end + def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) + safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode) + end - def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) - safe_provide_metric(:histogram, name, docstring, base_labels, buckets) - end + def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) + safe_provide_metric(:histogram, name, docstring, base_labels, buckets) + end - private + private - def safe_provide_metric(method, name, *args) - metric = provide_metric(name) - return metric if metric + def safe_provide_metric(method, name, *args) + metric = provide_metric(name) + return metric if metric - PROVIDER_MUTEX.synchronize do - provide_metric(name) || registry.method(method).call(name, *args) + PROVIDER_MUTEX.synchronize do + provide_metric(name) || registry.method(method).call(name, *args) + end end - end - def provide_metric(name) - if prometheus_metrics_enabled? - registry.get(name) - else - NullMetric.new + def provide_metric(name) + if prometheus_metrics_enabled? + registry.get(name) + else + NullMetric.instance + end end - end - def prometheus_metrics_enabled_unmemoized - metrics_folder_present? && current_application_settings[:prometheus_metrics_enabled] || false + def prometheus_metrics_enabled_unmemoized + metrics_folder_present? && Gitlab::CurrentSettings.prometheus_metrics_enabled || false + end end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 3da474fc1ec..b600e8a2a50 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -3,6 +3,14 @@ module Gitlab module Subscribers # Class for tracking the rendering timings of views. class ActionView < ActiveSupport::Subscriber + include Gitlab::Metrics::Methods + define_histogram :gitlab_view_rendering_duration_seconds do + docstring 'View rendering time' + base_labels Transaction::BASE_LABELS.merge({ path: nil }) + buckets [0.001, 0.01, 0.1, 1, 10.0] + with_feature :prometheus_metrics_view_instrumentation + end + attach_to :action_view SERIES = 'views'.freeze @@ -15,30 +23,18 @@ module Gitlab private - def metric_view_rendering_duration_seconds - @metric_view_rendering_duration_seconds ||= Gitlab::Metrics.histogram( - :gitlab_view_rendering_duration_seconds, - 'View rendering time', - Transaction::BASE_LABELS.merge({ path: nil }), - [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] - ) - end - def track(event) values = values_for(event) tags = tags_for(event) - metric_view_rendering_duration_seconds.observe( - current_transaction.labels.merge(tags), - event.duration - ) + self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration) current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end def relative_path(path) - path.gsub(/^#{Rails.root.to_s}\/?/, '') + path.gsub(%r{^#{Rails.root.to_s}/?}, '') end def values_for(event) diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index ead1acb8d44..4b3e8d0a6a0 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -3,12 +3,13 @@ module Gitlab module Subscribers # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber + include Gitlab::Metrics::Methods attach_to :active_record def sql(event) return unless current_transaction - metric_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) + self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) current_transaction.increment(:sql_count, 1, false) @@ -16,17 +17,14 @@ module Gitlab private - def current_transaction - Transaction.current + define_histogram :gitlab_sql_duration_seconds do + docstring 'SQL time' + base_labels Transaction::BASE_LABELS + buckets [0.001, 0.01, 0.1, 1.0, 10.0] end - def metric_sql_duration_seconds - @metric_sql_duration_seconds ||= Gitlab::Metrics.histogram( - :gitlab_sql_duration_seconds, - 'SQL time', - Transaction::BASE_LABELS, - [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] - ) + def current_transaction + Transaction.current end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index e7975c023a9..45b9e14ba55 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -2,11 +2,12 @@ module Gitlab module Metrics # Class for storing metrics information of a single transaction. class Transaction + include Gitlab::Metrics::Methods + # base labels shared among all transactions BASE_LABELS = { controller: nil, action: nil }.freeze THREAD_KEY = :_gitlab_metrics_transaction - METRICS_MUTEX = Mutex.new # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events'.freeze @@ -54,8 +55,8 @@ module Gitlab @memory_after = System.memory_usage @finished_at = System.monotonic_time - self.class.metric_transaction_duration_seconds.observe(labels, duration) - self.class.metric_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0) + self.class.gitlab_transaction_duration_seconds.observe(labels, duration) + self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0) Thread.current[THREAD_KEY] = nil end @@ -72,7 +73,7 @@ module Gitlab # event_name - The name of the event (e.g. "git_push"). # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) - self.class.metric_event_counter(event_name, tags).increment(tags.merge(labels)) + self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: tags).increment(tags.merge(labels)) @metrics << Metric.new(EVENT_SERIES, { count: 1 }, tags.merge(event: event_name), :event) end @@ -86,12 +87,12 @@ module Gitlab end def increment(name, value, use_prometheus = true) - self.class.metric_transaction_counter(name).increment(labels, value) if use_prometheus + self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus @values[name] += value end def set(name, value, use_prometheus = true) - self.class.metric_transaction_gauge(name).set(labels, value) if use_prometheus + self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus @values[name] = value end @@ -136,64 +137,28 @@ module Gitlab "#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty? end - def self.metric_transaction_duration_seconds - return @metric_transaction_duration_seconds if @metric_transaction_duration_seconds - - METRICS_MUTEX.synchronize do - @metric_transaction_duration_seconds ||= Gitlab::Metrics.histogram( - :gitlab_transaction_duration_seconds, - 'Transaction duration', - BASE_LABELS, - [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] - ) - end - end - - def self.metric_transaction_allocated_memory_bytes - return @metric_transaction_allocated_memory_bytes if @metric_transaction_allocated_memory_bytes - - METRICS_MUTEX.synchronize do - @metric_transaction_allocated_memory_bytes ||= Gitlab::Metrics.histogram( - :gitlab_transaction_allocated_memory_bytes, - 'Transaction allocated memory bytes', - BASE_LABELS, - [1000, 10000, 20000, 500000, 1000000, 2000000, 5000000, 10000000, 20000000, 100000000] - ) - end + define_histogram :gitlab_transaction_duration_seconds do + docstring 'Transaction duration' + base_labels BASE_LABELS + buckets [0.001, 0.01, 0.1, 1.0, 10.0] end - def self.metric_event_counter(event_name, tags) - return @metric_event_counters[event_name] if @metric_event_counters&.has_key?(event_name) - - METRICS_MUTEX.synchronize do - @metric_event_counters ||= {} - @metric_event_counters[event_name] ||= Gitlab::Metrics.counter( - "gitlab_transaction_event_#{event_name}_total".to_sym, - "Transaction event #{event_name} counter", - tags.merge(BASE_LABELS) - ) - end - end - - def self.metric_transaction_counter(name) - return @metric_transaction_counters[name] if @metric_transaction_counters&.has_key?(name) - - METRICS_MUTEX.synchronize do - @metric_transaction_counters ||= {} - @metric_transaction_counters[name] ||= Gitlab::Metrics.counter( - "gitlab_transaction_#{name}_total".to_sym, "Transaction #{name} counter", BASE_LABELS - ) - end + define_histogram :gitlab_transaction_allocated_memory_bytes do + docstring 'Transaction allocated memory bytes' + base_labels BASE_LABELS + buckets [100, 1000, 10000, 100000, 1000000, 10000000] + with_feature :prometheus_metrics_transaction_allocated_memory end - def self.metric_transaction_gauge(name) - return @metric_transaction_gauges[name] if @metric_transaction_gauges&.has_key?(name) + def self.transaction_metric(name, type, prefix: nil, tags: {}) + metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym + fetch_metric(type, metric_name) do + docstring "Transaction #{prefix}#{name} #{type}" + base_labels tags.merge(BASE_LABELS) - METRICS_MUTEX.synchronize do - @metric_transaction_gauges ||= {} - @metric_transaction_gauges[name] ||= Gitlab::Metrics.gauge( - "gitlab_transaction_#{name}".to_sym, "Transaction gauge #{name}", BASE_LABELS, :livesum - ) + if type == :gauge + multiprocess_mode :livesum + end end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index c6a56277922..1a570f480c6 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -4,7 +4,6 @@ module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper - include Gitlab::CurrentSettings PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -42,7 +41,7 @@ module Gitlab project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh' + repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" @@ -56,12 +55,12 @@ module Gitlab end def strip_url(url) - url.gsub(/\Ahttps?:\/\//, '') + url.gsub(%r{\Ahttps?://}, '') end def project_path(request) path_info = request.env["PATH_INFO"] - path_info.sub!(/^\//, '') + path_info.sub!(%r{^/}, '') project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) return unless project_path_match diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index fee741b47be..cc1e92480be 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -47,6 +47,7 @@ module Gitlab else value = decorate_params_value(value, @request.params[key], tmp_path) end + @request.update_param(key, value) end @@ -60,6 +61,7 @@ module Gitlab unless path_hash.is_a?(Hash) && path_hash.count == 1 raise "invalid path: #{path_hash.inspect}" end + path_key, path_value = path_hash.first unless value_hash.is_a?(Hash) && value_hash[path_key] diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb index 85ffa8aca68..aa1e9dc0fdb 100644 --- a/lib/gitlab/middleware/static.rb +++ b/lib/gitlab/middleware/static.rb @@ -1,7 +1,7 @@ module Gitlab module Middleware class Static < ActionDispatch::Static - UPLOADS_REGEX = /\A\/uploads(\/|\z)/.freeze + UPLOADS_REGEX = %r{\A/uploads(/|\z)}.freeze def call(env) return @app.call(env) if env['PATH_INFO'] =~ UPLOADS_REGEX diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index c22d0a84860..43921a8c1c0 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -37,6 +37,7 @@ module Gitlab else per_page - first_collection_last_page_size end + hash[page] = second_collection.page(second_collection_page) .per(per_page - paginated_first_collection(page).size) .padding(offset) diff --git a/lib/gitlab/o_auth.rb b/lib/gitlab/o_auth.rb new file mode 100644 index 00000000000..5ad8d83bd6e --- /dev/null +++ b/lib/gitlab/o_auth.rb @@ -0,0 +1,6 @@ +module Gitlab + module OAuth + SignupDisabledError = Class.new(StandardError) + SigninDisabledForProviderError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index d33f33d192f..e40a001d20c 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -5,8 +5,6 @@ # module Gitlab module OAuth - SignupDisabledError = Class.new(StandardError) - class User attr_accessor :auth_hash, :gl_user @@ -29,7 +27,8 @@ module Gitlab end def save(provider = 'OAuth') - unauthorized_to_create unless gl_user + raise SigninDisabledForProviderError if oauth_provider_disabled? + raise SignupDisabledError unless gl_user block_after_save = needs_blocking? @@ -56,7 +55,7 @@ module Gitlab user ||= find_or_build_ldap_user if auto_link_ldap_user? user ||= build_new_user if signup_enabled? - user.external = true if external_provider? && user + user.external = true if external_provider? && user&.new_record? user end @@ -226,8 +225,10 @@ module Gitlab Gitlab::AppLogger end - def unauthorized_to_create - raise SignupDisabledError + def oauth_provider_disabled? + Gitlab::CurrentSettings.current_application_settings + .disabled_oauth_sign_in_sources + .include?(auth_hash.provider) end end end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index e73245b82c1..6c2b2036074 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,18 +1,17 @@ module Gitlab module PerformanceBar - extend Gitlab::CurrentSettings - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze EXPIRY_TIME = 5.minutes def self.enabled?(user = nil) + return true if Rails.env.development? return false unless user && allowed_group_id allowed_user_ids.include?(user.id) end def self.allowed_group_id - current_application_settings.performance_bar_allowed_group_id + Gitlab::CurrentSettings.performance_bar_allowed_group_id end def self.allowed_user_ids diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index 4780675a492..fe4bdfe3831 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,12 +1,10 @@ module Gitlab class PollingInterval - extend Gitlab::CurrentSettings - HEADER_NAME = 'Poll-Interval'.freeze def self.set_header(response, interval:) if polling_enabled? - multiplier = current_application_settings.polling_interval_multiplier + multiplier = Gitlab::CurrentSettings.polling_interval_multiplier value = (interval * multiplier).to_i else value = -1 @@ -16,7 +14,7 @@ module Gitlab end def self.polling_enabled? - !current_application_settings.polling_interval_multiplier.zero? + !Gitlab::CurrentSettings.polling_interval_multiplier.zero? end end end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index 4bc5cda8cb5..b9832a724c4 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,7 +5,17 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil, vars = {}) + Result = Struct.new(:cmd, :stdout, :stderr, :status, :duration) + + # Returns [stdout + stderr, status] + def popen(cmd, path = nil, vars = {}, &block) + result = popen_with_detail(cmd, path, vars, &block) + + [result.stdout << result.stderr, result.status&.exitstatus] + end + + # Returns Result + def popen_with_detail(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end @@ -18,18 +28,21 @@ module Gitlab FileUtils.mkdir_p(path) end - cmd_output = "" - cmd_status = 0 + cmd_stdout = '' + cmd_stderr = '' + cmd_status = nil + start = Time.now + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| yield(stdin) if block_given? stdin.close - cmd_output << stdout.read - cmd_output << stderr.read - cmd_status = wait_thr.value.exitstatus + cmd_stdout = stdout.read + cmd_stderr = stderr.read + cmd_status = wait_thr.value end - [cmd_output, cmd_status] + Result.new(cmd, cmd_stdout, cmd_stderr, cmd_status, Time.now - start) end end end diff --git a/lib/gitlab/popen/runner.rb b/lib/gitlab/popen/runner.rb new file mode 100644 index 00000000000..f44035a48bb --- /dev/null +++ b/lib/gitlab/popen/runner.rb @@ -0,0 +1,46 @@ +module Gitlab + module Popen + class Runner + attr_reader :results + + def initialize + @results = [] + end + + def run(commands, &block) + commands.each do |cmd| + # yield doesn't support blocks, so we need to use a block variable + block.call(cmd) do # rubocop:disable Performance/RedundantBlockCall + cmd_result = Gitlab::Popen.popen_with_detail(cmd) + + results << cmd_result + + cmd_result + end + end + end + + def all_success_and_clean? + all_success? && all_stderr_empty? + end + + def all_success? + results.all? { |result| result.status.success? } + end + + def all_stderr_empty? + results.all? { |result| result.stderr.empty? } + end + + def failed_results + results.reject { |result| result.status.success? } + end + + def warned_results + results.select do |result| + result.status.success? && !result.stderr.empty? + end + end + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb new file mode 100644 index 00000000000..95d94b3cc68 --- /dev/null +++ b/lib/gitlab/profiler.rb @@ -0,0 +1,142 @@ +# coding: utf-8 +module Gitlab + module Profiler + FILTERED_STRING = '[FILTERED]'.freeze + + IGNORE_BACKTRACES = %w[ + lib/gitlab/i18n.rb + lib/gitlab/request_context.rb + config/initializers + lib/gitlab/database/load_balancing/ + lib/gitlab/etag_caching/ + lib/gitlab/metrics/ + lib/gitlab/middleware/ + lib/gitlab/performance_bar/ + lib/gitlab/request_profiler/ + lib/gitlab/profiler.rb + ].freeze + + # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) + # and returns the ruby-prof profile result. Formatting that result is the + # caller's responsibility. Requests are GET requests unless post_data is + # passed. + # + # Optional arguments: + # - logger: will be used for SQL logging, including a summary at the end of + # the log file of the total time spent per model class. + # + # - post_data: a string of raw POST data to use. Changes the HTTP verb to + # POST. + # + # - user: a user to authenticate as. Only works if the user has a valid + # personal access token. + # + # - private_token: instead of providing a user instance, the token can be + # given as a string. Takes precedence over the user option. + def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil) + app = ActionDispatch::Integration::Session.new(Rails.application) + verb = :get + headers = {} + + if post_data + verb = :post + headers['Content-Type'] = 'application/json' + end + + if user + private_token ||= user.personal_access_tokens.active.pluck(:token).first + end + + headers['Private-Token'] = private_token if private_token + logger = create_custom_logger(logger, private_token: private_token) + + RequestStore.begin! + + # Make an initial call for an asset path in development mode to avoid + # sprockets dominating the profiler output. + ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development? + + # Rails loads internationalization files lazily the first time a + # translation is needed. Running this prevents this overhead from showing + # up in profiles. + ::I18n.t('.')[:test_string] + + # Remove API route mounting from the profile. + app.get('/api/v4/users') + + result = with_custom_logger(logger) do + RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend + end + + RequestStore.end! + + log_load_times_by_model(logger) + + result + end + + def self.create_custom_logger(logger, private_token: nil) + return unless logger + + logger.dup.tap do |new_logger| + new_logger.instance_variable_set(:@private_token, private_token) + + class << new_logger + attr_reader :load_times_by_model, :private_token + + def debug(message, *) + message.gsub!(private_token, FILTERED_STRING) if private_token + + _, type, time = *message.match(/(\w+) Load \(([0-9.]+)ms\)/) + + if type && time + @load_times_by_model ||= {} + @load_times_by_model[type] ||= 0 + @load_times_by_model[type] += time.to_f + end + + super + + backtrace = Rails.backtrace_cleaner.clean(caller) + + backtrace.each do |caller_line| + next if caller_line.match(Regexp.union(IGNORE_BACKTRACES)) + + stripped_caller_line = caller_line.sub("#{Rails.root}/", '') + + super(" ↳ #{stripped_caller_line}") + end + end + end + end + end + + def self.with_custom_logger(logger) + original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging + original_activerecord_logger = ActiveRecord::Base.logger + original_actioncontroller_logger = ActionController::Base.logger + + if logger + ActiveSupport::LogSubscriber.colorize_logging = false + ActiveRecord::Base.logger = logger + ActionController::Base.logger = logger + end + + result = yield + + ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging + ActiveRecord::Base.logger = original_activerecord_logger + ActionController::Base.logger = original_actioncontroller_logger + + result + end + + def self.log_load_times_by_model(logger) + return unless logger.respond_to?(:load_times_by_model) + + logger.load_times_by_model.to_a.sort_by(&:last).reverse.each do |(model, time)| + logger.info("#{model} total: #{time.round(2)}ms") + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index e2662fc362b..4823f703ba4 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -20,7 +20,7 @@ module Gitlab when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) else - super + super(scope, page, false) end end @@ -44,25 +44,20 @@ module Gitlab ref = nil filename = nil basename = nil + data = "" startline = 0 - result.each_line.each_with_index do |line, index| - matches = line.match(/^(?<ref>[^:]*):(?<filename>.*):(?<startline>\d+):/) - if matches + result.strip.each_line.each_with_index do |line, index| + prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>.*)\x00(?<startline>\d+)\x00/)&.tap do |matches| ref = matches[:ref] filename = matches[:filename] startline = matches[:startline] startline = startline.to_i - index extname = Regexp.escape(File.extname(filename)) basename = filename.sub(/#{extname}$/, '') - break end - end - - data = "" - result.each_line do |line| - data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') + data << line.sub(prefix.to_s, '') end FoundBlob.new( diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 09fa14764e6..2819c7d062c 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,14 +1,12 @@ module Gitlab module ProtocolAccess - extend Gitlab::CurrentSettings - def self.allowed?(protocol) if protocol == 'web' true - elsif current_application_settings.enabled_git_access_protocol.blank? + elsif Gitlab::CurrentSettings.enabled_git_access_protocol.blank? true else - protocol == current_application_settings.enabled_git_access_protocol + protocol == Gitlab::CurrentSettings.enabled_git_access_protocol end end end diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb new file mode 100644 index 00000000000..f64f1757144 --- /dev/null +++ b/lib/gitlab/query_limiting.rb @@ -0,0 +1,36 @@ +module Gitlab + module QueryLimiting + # Returns true if we should enable tracking of query counts. + # + # This is only enabled in production/staging if we're running on GitLab.com. + # This ensures we don't produce any errors that users can't do anything + # about themselves. + def self.enable? + Gitlab.com? || Rails.env.development? || Rails.env.test? + end + + # Allows the current request to execute any number of SQL queries. + # + # This method should _only_ be used when there's a corresponding issue to + # reduce the number of queries. + # + # The issue URL is only meant to push developers into creating an issue + # instead of blindly whitelisting offending blocks of code. + def self.whitelist(issue_url) + return unless enable_whitelist? + + unless issue_url.start_with?('https://') + raise( + ArgumentError, + 'You must provide a valid issue URL in order to whitelist a block of code' + ) + end + + Transaction&.current&.whitelisted = true + end + + def self.enable_whitelist? + Rails.env.development? || Rails.env.test? + end + end +end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb new file mode 100644 index 00000000000..66049c94ec6 --- /dev/null +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -0,0 +1,11 @@ +module Gitlab + module QueryLimiting + class ActiveSupportSubscriber < ActiveSupport::Subscriber + attach_to :active_record + + def sql(*) + Transaction.current&.increment + end + end + end +end diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb new file mode 100644 index 00000000000..949ae79a047 --- /dev/null +++ b/lib/gitlab/query_limiting/middleware.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module QueryLimiting + # Middleware for reporting (or raising) when a request performs more than a + # certain amount of database queries. + class Middleware + CONTROLLER_KEY = 'action_controller.instance'.freeze + ENDPOINT_KEY = 'api.endpoint'.freeze + + def initialize(app) + @app = app + end + + def call(env) + transaction, retval = Transaction.run do + @app.call(env) + end + + transaction.action = action_name(env) + transaction.act_upon_results + + retval + end + + def action_name(env) + if env[CONTROLLER_KEY] + action_for_rails(env) + elsif env[ENDPOINT_KEY] + action_for_grape(env) + end + end + + private + + def action_for_rails(env) + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + + if controller.content_type == 'text/html' + action + else + "#{action} (#{controller.content_type})" + end + end + + def action_for_grape(env) + endpoint = env[ENDPOINT_KEY] + route = endpoint.route rescue nil + + "#{route.request_method} #{route.path}" if route + end + end + end +end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb new file mode 100644 index 00000000000..3cbafadb0d0 --- /dev/null +++ b/lib/gitlab/query_limiting/transaction.rb @@ -0,0 +1,83 @@ +module Gitlab + module QueryLimiting + class Transaction + THREAD_KEY = :__gitlab_query_counts_transaction + + attr_accessor :count, :whitelisted + + # The name of the action (e.g. `UsersController#show`) that is being + # executed. + attr_accessor :action + + # The maximum number of SQL queries that can be executed in a request. For + # the sake of keeping things simple we hardcode this value here, it's not + # supposed to be changed very often anyway. + THRESHOLD = 100 + + # Error that is raised whenever exceeding the maximum number of queries. + ThresholdExceededError = Class.new(StandardError) + + def self.current + Thread.current[THREAD_KEY] + end + + # Starts a new transaction and returns it and the blocks' return value. + # + # Example: + # + # transaction, retval = Transaction.run do + # 10 + # end + # + # retval # => 10 + def self.run + transaction = new + Thread.current[THREAD_KEY] = transaction + + [transaction, yield] + ensure + Thread.current[THREAD_KEY] = nil + end + + def initialize + @action = nil + @count = 0 + @whitelisted = false + end + + # Sends a notification based on the number of executed SQL queries. + def act_upon_results + return unless threshold_exceeded? + + error = ThresholdExceededError.new(error_message) + + if raise_error? + raise(error) + else + # Raven automatically logs to the Rails log if disabled, thus we don't + # need to manually log anything in case Sentry support is not enabled. + Raven.capture_exception(error) + end + end + + def increment + @count += 1 unless whitelisted + end + + def raise_error? + Rails.env.test? + end + + def threshold_exceeded? + count > THRESHOLD + end + + def error_message + header = 'Too many SQL queries were executed' + header += " in #{action}" if action + + "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + end + end + end +end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 3ebfa3bd4b8..c0878a34fb1 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -126,6 +126,7 @@ module Gitlab command << match_data[1] unless match_data[1].empty? commands << command end + content = substitution.perform_substitution(self, content) end diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index 3f52402b31f..7328c517a30 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -9,7 +9,7 @@ module Gitlab # if date doesn't present return time with current date # in other cases return nil class SpendTimeAndDateSeparator - DATE_REGEX = /(\d{2,4}[\/\-.]\d{1,2}[\/\-.]\d{1,2})/ + DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})} def initialize(spend_command_arg) @spend_arg = spend_command_arg diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index c463dd487a0..c9efa28d7e7 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,12 +1,10 @@ module Gitlab module Recaptcha - extend Gitlab::CurrentSettings - def self.load_configurations! - if current_application_settings.recaptcha_enabled + if Gitlab::CurrentSettings.recaptcha_enabled ::Recaptcha.configure do |config| - config.public_key = current_application_settings.recaptcha_site_key - config.private_key = current_application_settings.recaptcha_private_key + config.public_key = Gitlab::CurrentSettings.recaptcha_site_key + config.private_key = Gitlab::CurrentSettings.recaptcha_private_key end true @@ -14,7 +12,7 @@ module Gitlab end def self.enabled? - current_application_settings.recaptcha_enabled + Gitlab::CurrentSettings.recaptcha_enabled end end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 9bf019b72e6..a991933e910 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,5 +1,5 @@ # please require all dependencies below: -require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) +require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present? module Gitlab module Redis diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 8ad06480575..4178b436acf 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -24,6 +24,7 @@ module Gitlab # the pool will be used in a multi-threaded context size += Sidekiq.options[:concurrency] end + size end @@ -104,6 +105,7 @@ module Gitlab db_numbers = queries["db"] if queries.key?("db") config[:db] = db_numbers[0].to_i if db_numbers.any? end + config else redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 2c7b8af83f2..7ab85e1c35c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -37,7 +37,7 @@ module Gitlab end def environment_name_regex_chars - 'a-zA-Z0-9_/\\$\\{\\}\\. -' + 'a-zA-Z0-9_/\\$\\{\\}\\. \\-' end def environment_name_regex @@ -67,7 +67,7 @@ module Gitlab end def build_trace_section_regex - @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/.freeze + @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze end end end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 3591fa9145e..79265cf952d 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -30,7 +30,7 @@ module Gitlab raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end - result.sub(/\A\/*/, '') + result.sub(%r{\A/*}, '') end def self.find_project(project_path) diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7037e2e61cc..7362514167f 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -40,19 +40,21 @@ module Gitlab @default_project_filter = default_project_filter end - def objects(scope, page = nil) - case scope - when 'projects' - projects.page(page).per(per_page) - when 'issues' - issues.page(page).per(per_page) - when 'merge_requests' - merge_requests.page(page).per(per_page) - when 'milestones' - milestones.page(page).per(per_page) - else - Kaminari.paginate_array([]).page(page).per(per_page) - end + def objects(scope, page = nil, without_count = true) + collection = case scope + when 'projects' + projects.page(page).per(per_page) + when 'issues' + issues.page(page).per(per_page) + when 'merge_requests' + merge_requests.page(page).per(per_page) + when 'milestones' + milestones.page(page).per(per_page) + else + Kaminari.paginate_array([]).page(page).per(per_page) + end + + without_count ? collection.without_count : collection end def projects_count @@ -71,18 +73,49 @@ module Gitlab @milestones_count ||= milestones.count end + def limited_projects_count + @limited_projects_count ||= projects.limit(count_limit).count + end + + def limited_issues_count + return @limited_issues_count if @limited_issues_count + + # By default getting limited count (e.g. 1000+) is fast on issuable + # collections except for issues, where filtering both not confidential + # and confidential issues user has access to, is too complex. + # It's faster to try to fetch all public issues first, then only + # if necessary try to fetch all issues. + sum = issues(public_only: true).limit(count_limit).count + @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum + end + + def limited_merge_requests_count + @limited_merge_requests_count ||= merge_requests.limit(count_limit).count + end + + def limited_milestones_count + @limited_milestones_count ||= milestones.limit(count_limit).count + end + def single_commit_result? false end + def count_limit + 1001 + end + private def projects limit_projects.search(query) end - def issues - issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) + def issues(finder_params = {}) + issues = IssuesFinder.new(current_user, finder_params).execute + unless default_project_filter + issues = issues.where(project_id: project_ids_relation) + end issues = if query =~ /#(\d+)\z/ @@ -91,13 +124,13 @@ module Gitlab issues.full_search(query) end - issues.order('updated_at DESC') + issues.reorder('updated_at DESC') end def milestones milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) - milestones.order('updated_at DESC') + milestones.reorder('updated_at DESC') end def merge_requests @@ -112,7 +145,8 @@ module Gitlab else merge_requests.full_search(query) end - merge_requests.order('updated_at DESC') + + merge_requests.reorder('updated_at DESC') end def default_scope diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 30df7e4a831..98f005cb61b 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -5,10 +5,16 @@ module DeliverNever end end +module MuteNotifications + def new_note(note) + end +end + module Gitlab class Seeder def self.quiet - mute_mailer unless Rails.env.test? + mute_notifications + mute_mailer SeedFu.quiet = true @@ -18,6 +24,10 @@ module Gitlab puts "\nOK".color(:green) end + def self.mute_notifications + NotificationService.prepend(MuteNotifications) + end + def self.mute_mailer ActionMailer::MessageDelivery.prepend(DeliverNever) end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 159d0e7952e..4a22fc80f75 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,9 +1,7 @@ module Gitlab module Sentry - extend Gitlab::CurrentSettings - def self.enabled? - Rails.env.production? && current_application_settings.sentry_enabled? + Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled? end def self.context(current_user = nil) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb new file mode 100644 index 00000000000..e90a90508a2 --- /dev/null +++ b/lib/gitlab/setup_helper.rb @@ -0,0 +1,61 @@ +module Gitlab + module SetupHelper + class << self + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only generate a configuration for the most common and simplest case: when + # we have exactly one Gitaly process and we are sure it is running locally + # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. + def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + + config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages } + config[:auth] = { token: 'secret' } if Rails.env.test? + config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby + config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + + TOML.dump(config) + end + + # rubocop:disable Rails/Output + def create_gitaly_configuration(dir, force: false) + config_path = File.join(dir, 'config.toml') + FileUtils.rm_f(config_path) if force + + File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| + f.puts gitaly_configuration_toml(dir) + end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end + # rubocop:enable Rails/Output + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 9cdd3d22f18..f4a41dc3eda 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -71,7 +71,6 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') @@ -100,8 +99,12 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) + if url.start_with?('.', '/') + raise Error.new("don't use disk paths with import_repository: #{url.inspect}") + end + # The timeout ensures the subprocess won't hang forever cmd = gitlab_projects(storage, "#{name}.git") success = cmd.import_project(url, git_timeout) @@ -122,11 +125,10 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout) else storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) @@ -134,7 +136,10 @@ module Gitlab end end - # Move repository + # Move repository reroutes to mv_directory which is an alias for + # mv_namespace. Given the underlying implementation is a move action, + # indescriminate of what the folders might be. + # # storage - project's storage path # path - project disk path # new_path - new project disk path @@ -142,9 +147,11 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) - gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git") + return false if path.empty? || new_path.empty? + + !!mv_directory(storage, "#{path}.git", "#{new_path}.git") end # Fork repository to new path @@ -156,13 +163,15 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # - # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") end - # Remove repository from file system + # Removes a repository from file system, using rm_diretory which is an alias + # for rm_namespace. Given the underlying implementation removes the name + # passed as second argument on the passed storage. # # storage - project's storage path # name - project disk path @@ -170,9 +179,14 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) - gitlab_projects(storage, "#{name}.git").rm_project + return false if name.empty? + + !!rm_directory(storage, "#{name}.git") + rescue ArgumentError => e + Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") + false end # Add new key to gitlab-shell @@ -181,6 +195,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'add-key', key_id, self.class.strip_key(key_content)]) end @@ -190,6 +206,8 @@ module Gitlab # Ex. # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } def batch_add_keys(&block) + return unless self.authorized_keys_enabled? + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| yield(KeyAdder.new(io)) end @@ -200,10 +218,11 @@ module Gitlab # Ex. # remove_key("key-342", "sha-rsa ...") # - def remove_key(key_id, key_content) + def remove_key(key_id, key_content = nil) + return unless self.authorized_keys_enabled? + args = [gitlab_shell_keys_path, 'rm-key', key_id] args << key_content if key_content - gitlab_shell_fast_execute(args) end @@ -213,15 +232,67 @@ module Gitlab # remove_all_keys # def remove_all_keys + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) end + # Remove ssh keys from gitlab shell that are not in the DB + # + # Ex. + # remove_keys_not_found_in_db + # + def remove_keys_not_found_in_db + return unless self.authorized_keys_enabled? + + Rails.logger.info("Removing keys not found in DB") + + batch_read_key_ids do |ids_in_file| + ids_in_file.uniq! + keys_in_db = Key.where(id: ids_in_file) + + next unless ids_in_file.size > keys_in_db.count # optimization + + ids_to_remove = ids_in_file - keys_in_db.pluck(:id) + ids_to_remove.each do |id| + Rails.logger.info("Removing key-#{id} not found in DB") + remove_key("key-#{id}") + end + end + end + + # Iterate over all ssh key IDs from gitlab shell, in batches + # + # Ex. + # batch_read_key_ids { |batch| keys = Key.where(id: batch) } + # + def batch_read_key_ids(batch_size: 100, &block) + return unless self.authorized_keys_enabled? + + list_key_ids do |key_id_stream| + key_id_stream.lazy.each_slice(batch_size) do |lines| + key_ids = lines.map { |l| l.chomp.to_i } + yield(key_ids) + end + end + end + + # Stream all ssh key IDs from gitlab shell, separated by newlines + # + # Ex. + # list_key_ids + # + def list_key_ids(&block) + return unless self.authorized_keys_enabled? + + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block) + end + # Add empty directory for storing repositories # # Ex. # add_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| if enabled @@ -243,7 +314,6 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| if enabled @@ -255,13 +325,13 @@ module Gitlab rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end + alias_method :rm_directory, :rm_namespace # Move namespace directory inside repositories storage # # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| if enabled @@ -275,6 +345,7 @@ module Gitlab rescue GRPC::InvalidArgument false end + alias_method :mv_directory, :mv_namespace def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" @@ -306,47 +377,6 @@ module Gitlab end end - # Push branch to remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names to push - # forced - should we use --force flag - # - # Ex. - # push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature']) - # - def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.push_branches(remote_name, git_timeout, forced, branch_names) - - raise Error, cmd.output unless success - - success - end - - # Delete branch from remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names - # - # Ex. - # delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature']) - # - def delete_remote_branches(storage, project_name, remote_name, branch_names) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.delete_remote_branches(remote_name, branch_names) - - raise Error, cmd.output unless success - - success - end - protected def gitlab_shell_path @@ -375,6 +405,14 @@ module Gitlab File.join(gitlab_shell_path, 'bin', 'gitlab-keys') end + def authorized_keys_enabled? + # Return true if nil to ensure the authorized_keys methods work while + # fixing the authorized_keys file during migration. + return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil? + + Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + end + private def gitlab_projects(shard_path, disk_path) diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb index 8a3e1a5e5bf..89072b01f2e 100644 --- a/lib/gitlab/sherlock/file_sample.rb +++ b/lib/gitlab/sherlock/file_sample.rb @@ -16,7 +16,7 @@ module Gitlab end def relative_path - @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + @relative_path ||= @file.gsub(%r{^#{Rails.root.to_s}/?}, '') end def to_param diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index 687332fc5fc..4c88e33699a 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -2,7 +2,7 @@ module Gitlab module Sherlock # Rack middleware used for tracking request metrics. class Middleware - CONTENT_TYPES = /text\/html|application\/json/i + CONTENT_TYPES = %r{text/html|application/json}i IGNORE_PATHS = %r{^/sherlock} diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 948bf5e6528..02ddc3f47eb 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -4,7 +4,7 @@ module Gitlab attr_reader :id, :query, :started_at, :finished_at, :backtrace # SQL identifiers that should be prefixed with newlines. - PREFIX_NEWLINE = / + PREFIX_NEWLINE = %r{ \s+(FROM |(LEFT|RIGHT)?INNER\s+JOIN |(LEFT|RIGHT)?OUTER\s+JOIN @@ -13,7 +13,7 @@ module Gitlab |GROUP\s+BY |ORDER\s+BY |LIMIT - |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + |OFFSET)\s+}ix # Vim indent breaks when this is on a newline :< # Creates a new Query using a String and a separate Array of bindings. # diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index b85f70e450e..4f86b3e8f73 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -16,7 +16,7 @@ module Gitlab when 'snippet_blobs' snippet_blobs.page(page).per(per_page) else - super + super(scope, nil, false) end end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 89ca1298120..545e7c74f7e 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -21,6 +21,22 @@ module Gitlab technology(name)&.supported_sizes end + def self.sanitize(key_content) + ssh_type, *parts = key_content.strip.split + + return key_content if parts.empty? + + parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index| + content << part + + if Gitlab::SSHPublicKey.new(content).valid? + break [content, parts[index + 1]].compact.join(' ') # Add the comment part if present + elsif parts.size == index + 1 # return original content if we've reached the last element + break key_content + end + end + end + attr_reader :key_text, :key # Unqualified MD5 fingerprint for compatibility @@ -37,23 +53,23 @@ module Gitlab end def valid? - key.present? + key.present? && bits && technology.supported_sizes.include?(bits) end def type - technology.name if valid? + technology.name if key.present? end def bits - return unless valid? + return if key.blank? case type when :rsa - key.n.num_bits + key.n&.num_bits when :dsa - key.p.num_bits + key.p&.num_bits when :ecdsa - key.group.order.num_bits + key.group.order&.num_bits when :ed25519 256 else diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb index 04bf1bf1d26..9b64c8e033a 100644 --- a/lib/gitlab/storage_check/cli.rb +++ b/lib/gitlab/storage_check/cli.rb @@ -59,9 +59,11 @@ module Gitlab if response.skipped_shards.any? warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" end + if response.failing_shards.any? warnings << "Failing shards: #{response.failing_shards.join(', ')}" end + logger.warn(warnings.join(' - ')) if warnings.any? end end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb new file mode 100644 index 00000000000..34bee6fecbe --- /dev/null +++ b/lib/gitlab/task_helpers.rb @@ -0,0 +1,169 @@ +require 'rainbow/ext/string' +require 'gitlab/utils/strong_memoize' + +# rubocop:disable Rails/Output +module Gitlab + TaskFailedError = Class.new(StandardError) + TaskAbortedByUserError = Class.new(StandardError) + + module TaskHelpers + include Gitlab::Utils::StrongMemoize + + extend self + + # Ask if the user wants to continue + # + # Returns "yes" the user chose to continue + # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue + def ask_to_continue + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) + raise Gitlab::TaskAbortedByUserError unless answer == "yes" + end + + # Check which OS is running + # + # It will primarily use lsb_relase to determine the OS. + # It has fallbacks to Debian, SuSE, OS X and systems running systemd. + def os_name + os_name = run_command(%w(lsb_release -irs)) + os_name ||= + if File.readable?('/etc/system-release') + File.read('/etc/system-release') + elsif File.readable?('/etc/debian_version') + "Debian #{File.read('/etc/debian_version')}" + elsif File.readable?('/etc/SuSE-release') + File.read('/etc/SuSE-release') + elsif os_x_version = run_command(%w(sw_vers -productVersion)) + "Mac OS X #{os_x_version}" + elsif File.readable?('/etc/os-release') + File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] + end + + os_name.try(:squish!) + end + + # Prompt the user to input something + # + # message - the message to display before input + # choices - array of strings of acceptable answers or nil for any answer + # + # Returns the user's answer + def prompt(message, choices = nil) + begin + print(message) + answer = STDIN.gets.chomp + end while choices.present? && !choices.include?(answer) + answer + end + + # Runs the given command and matches the output against the given pattern + # + # Returns nil if nothing matched + # Returns the MatchData if the pattern matched + # + # see also #run_command + # see also String#match + def run_and_match(command, regexp) + run_command(command).try(:match, regexp) + end + + # Runs the given command + # + # Returns '' if the command was not found + # Returns the output of the command otherwise + # + # see also #run_and_match + def run_command(command) + output, _ = Gitlab::Popen.popen(command) + output + rescue Errno::ENOENT + '' # if the command does not exist, return an empty string + end + + # Runs the given command and raises a Gitlab::TaskFailedError exception if + # the command does not exit with 0 + # + # Returns the output of the command otherwise + def run_command!(command) + output, status = Gitlab::Popen.popen(command) + + raise Gitlab::TaskFailedError.new(output) unless status.zero? + + output + end + + def uid_for(user_name) + run_command(%W(id -u #{user_name})).chomp.to_i + end + + def gid_for(group_name) + Etc.getgrnam(group_name).gid + rescue ArgumentError # no group + "group #{group_name} doesn't exist" + end + + def gitlab_user + Gitlab.config.gitlab.user + end + + def gitlab_user? + strong_memoize(:is_gitlab_user) do + current_user = run_command(%w(whoami)).chomp + current_user == gitlab_user + end + end + + def warn_user_is_not_gitlab + return if gitlab_user? + + strong_memoize(:warned_user_not_gitlab) do + current_user = run_command(%w(whoami)).chomp + + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" + end + end + + def all_repos + Gitlab.config.repositories.storages.each_value do |repository_storage| + IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end + end + end + end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } + end + + def user_home + Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home + end + + def checkout_or_clone_version(version:, repo:, target_dir:) + version = + if version.starts_with?("=") + version.sub(/\A=/, '') # tag or branch + else + "v#{version}" # tag + end + + clone_repo(repo, target_dir) unless Dir.exist?(target_dir) + checkout_version(version, target_dir) + end + + def clone_repo(repo, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + end + + def checkout_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet origin #{version}]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout -f --quiet FETCH_HEAD --]) + end + end +end diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb index 4a8e3c2eee0..53333b9b06b 100644 --- a/lib/gitlab/testing/request_blocker_middleware.rb +++ b/lib/gitlab/testing/request_blocker_middleware.rb @@ -37,12 +37,14 @@ module Gitlab def call(env) increment_active_requests + if block_requests? block_request(env) else sleep 0.2 if slow_requests? @app.call(env) end + ensure decrement_active_requests end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index b290c716f97..76a1808c8ac 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -9,6 +9,7 @@ module Gitlab else block.call end + ensure model.record_timestamps = original_record_timestamps end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index 961df0468a4..024be6aca44 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -1,6 +1,3 @@ -require_relative "popen" -require_relative "version_info" - module Gitlab class Upgrader def execute @@ -12,6 +9,7 @@ module Gitlab puts "You are using the latest GitLab version" else puts "Newer GitLab version is available" + answer = if ARGV.first == "-y" "yes" else @@ -51,7 +49,7 @@ module Gitlab def fetch_git_tags remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) - remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) + remote_tags.split("\n").grep(%r{tags/v#{current_version.major}}) end def update_commands @@ -77,6 +75,7 @@ module Gitlab update_commands.each do |title, cmd| puts title puts " -> #{cmd.join(' ')}" + if system(env, *cmd) puts " -> OK" else diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index b5f41240529..7d7400bdabf 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,7 +1,7 @@ module Gitlab class UploadsTransfer < ProjectTransfer def root_dir - File.join(CarrierWave.root, FileUploader.base_dir) + FileUploader.root end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 2adcc9809b3..9d13d1d781f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,8 +1,6 @@ module Gitlab class UsageData class << self - include Gitlab::CurrentSettings - def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } end @@ -19,7 +17,7 @@ module Gitlab def license_usage_data usage_data = { - uuid: current_application_settings.uuid, + uuid: Gitlab::CurrentSettings.uuid, hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, active_user_count: User.active.count, @@ -79,9 +77,9 @@ module Gitlab def features_usage_data_ce { - signup: current_application_settings.allow_signup?, + signup: Gitlab::CurrentSettings.allow_signup?, ldap: Gitlab.config.ldap.enabled, - gravatar: current_application_settings.gravatar_enabled?, + gravatar: Gitlab::CurrentSettings.gravatar_enabled?, omniauth: Gitlab.config.omniauth.enabled, reply_by_email: Gitlab::IncomingEmail.enabled?, container_registry: Gitlab.config.registry.enabled, diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index d9a5af09f08..f357488ac61 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -16,8 +16,10 @@ module Gitlab def can_do_action?(action) return false unless can_access_git? - @permission_cache ||= {} - @permission_cache[action] ||= user.can?(action, project) + permission_cache[action] = + permission_cache.fetch(action) do + user.can?(action, project) + end end def cannot_do_action?(action) @@ -88,6 +90,10 @@ module Gitlab private + def permission_cache + @permission_cache ||= {} + end + def can_access_git? user && user.can?(:access_git) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index b3baaf036d8..fa22f0e37b2 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -27,6 +27,10 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end + def remove_line_breaks(str) + str.gsub(/\r?\n/, '') + end + def to_boolean(value) return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb new file mode 100644 index 00000000000..8bf6bcb1fe2 --- /dev/null +++ b/lib/gitlab/utils/override.rb @@ -0,0 +1,111 @@ +module Gitlab + module Utils + module Override + class Extension + def self.verify_class!(klass, method_name) + instance_method_defined?(klass, method_name) || + raise( + NotImplementedError.new( + "#{klass}\##{method_name} doesn't exist!")) + end + + def self.instance_method_defined?(klass, name, include_super: true) + klass.instance_methods(include_super).include?(name) || + klass.private_instance_methods(include_super).include?(name) + end + + attr_reader :subject + + def initialize(subject) + @subject = subject + end + + def add_method_name(method_name) + method_names << method_name + end + + def add_class(klass) + classes << klass + end + + def verify! + classes.each do |klass| + index = klass.ancestors.index(subject) + parents = klass.ancestors.drop(index + 1) + + method_names.each do |method_name| + parents.any? do |parent| + self.class.instance_method_defined?( + parent, method_name, include_super: false) + end || + raise( + NotImplementedError.new( + "#{klass}\##{method_name} doesn't exist!")) + end + end + end + + private + + def method_names + @method_names ||= [] + end + + def classes + @classes ||= [] + end + end + + # Instead of writing patterns like this: + # + # def f + # raise NotImplementedError unless defined?(super) + # + # true + # end + # + # We could write it like: + # + # extend ::Gitlab::Utils::Override + # + # override :f + # def f + # true + # end + # + # This would make sure we're overriding something. See: + # https://gitlab.com/gitlab-org/gitlab-ee/issues/1819 + def override(method_name) + return unless ENV['STATIC_VERIFICATION'] + + if is_a?(Class) + Extension.verify_class!(self, method_name) + else # We delay the check for modules + Override.extensions[self] ||= Extension.new(self) + Override.extensions[self].add_method_name(method_name) + end + end + + def included(base = nil) + return super if base.nil? # Rails concern, ignoring it + + super + + if base.is_a?(Class) # We could check for Class in `override` + # This could be `nil` if `override` was never called + Override.extensions[self]&.add_class(base) + end + end + + alias_method :prepended, :included + + def self.extensions + @extensions ||= {} + end + + def self.verify! + extensions.values.each(&:verify!) + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 11472ce6cce..2612208a927 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -5,7 +5,6 @@ # module Gitlab module VisibilityLevel - extend CurrentSettings extend ActiveSupport::Concern included do @@ -57,11 +56,17 @@ module Gitlab } end - def highest_allowed_level - restricted_levels = current_application_settings.restricted_visibility_levels + def allowed_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels - allowed_levels = self.values - restricted_levels - allowed_levels.max || PRIVATE + self.values - Array(restricted_levels) + end + + def closest_allowed_level(target_level) + highest_allowed_level = allowed_levels.select { |level| level <= target_level }.max + + # If all levels are restricted, fall back to PRIVATE + highest_allowed_level || PRIVATE end def allowed_for?(user, level) @@ -75,7 +80,7 @@ module Gitlab end def non_restricted_level?(level) - restricted_levels = current_application_settings.restricted_visibility_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels if restricted_levels.nil? true diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5ab6cd5a4ef..b3f8b0d174d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -34,7 +34,10 @@ module Gitlab feature_enabled = case action.to_s when 'git_receive_pack' - Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) + Gitlab::GitalyClient.feature_enabled?( + :post_receive_pack, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT + ) when 'git_upload_pack' true when 'info_refs' @@ -42,6 +45,7 @@ module Gitlab else raise "Unsupported action: #{action}" end + if feature_enabled params[:GitalyServer] = server end @@ -51,14 +55,14 @@ module Gitlab def lfs_upload_ok(oid, size) { - StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", + StoreLFSPath: LfsObjectUploader.workhorse_upload_path, LfsOid: oid, LfsSize: size } end def artifact_upload_ok - { TempPath: JobArtifactUploader.artifacts_upload_path } + { TempPath: JobArtifactUploader.workhorse_upload_path } end def send_git_blob(repository, blob) @@ -97,6 +101,9 @@ module Gitlab ) end + # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. + params['DisableCache'] = true if git_archive_cache_disabled? + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" @@ -140,8 +147,11 @@ module Gitlab end def send_artifacts_entry(build, entry) + file = build.artifacts_file + archive = file.file_storage? ? file.path : file.url + params = { - 'Archive' => build.artifacts_file.path, + 'Archive' => archive, 'Entry' => Base64.encode64(entry.to_s) } @@ -244,6 +254,10 @@ module Gitlab right_commit_id: diff_refs.head_sha } end + + def git_archive_cache_disabled? + ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled) + end end end end |