diff options
Diffstat (limited to 'lib/gitlab')
58 files changed, 680 insertions, 1073 deletions
diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 6c5d0788a0a..e7283b2f9e8 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -74,6 +74,10 @@ module Gitlab gl_user end + def bypass_two_factor? + false + end + protected def should_save? diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index c345a7e3f6c..3bc5e2864df 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -6,6 +6,17 @@ module Gitlab Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) end + def authn_context + response_object = auth_hash.extra[:response_object] + return nil if response_object.blank? + + document = response_object.decrypted_document + document ||= response_object.document + return nil if document.blank? + + extract_authn_context(document) + end + private def get_raw(key) @@ -13,6 +24,10 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end + + def extract_authn_context(document) + REXML::XPath.first(document, "//saml:AuthnStatement/saml:AuthnContext/saml:AuthnContextClassRef/text()").to_s + end end end end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 5fa9581f837..625dab7c6f4 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -7,6 +7,10 @@ module Gitlab Gitlab::Auth::OAuth::Provider.config_for('saml') end + def upstream_two_factor_authn_contexts + options.args[:upstream_two_factor_authn_contexts] + end + def groups options[:groups_attribute] end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index b8c84c37cd5..6c3b75f3eb0 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -34,6 +34,10 @@ module Gitlab gl_user.changed? || gl_user.identities.any?(&:changed?) end + def bypass_two_factor? + saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context) + end + protected def saml_config diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 914a9e48a2f..522c69a0bb1 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -54,7 +54,8 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless UntrackedFile.connection.table_exists?(table_name) + + unless ActiveRecord::Base.connection.data_source_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index ecc85f847d4..671b8e7e1b1 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -1,41 +1,6 @@ module Gitlab module Cache - # This module provides a simple way to cache values in RequestStore, - # and the cache key would be based on the class name, method name, - # optionally customized instance level values, optionally customized - # method level values, and optional method arguments. - # - # A simple example: - # - # class UserAccess - # extend Gitlab::Cache::RequestCache - # - # request_cache_key do - # [user&.id, project&.id] - # end - # - # request_cache def can_push_to_branch?(ref) - # # ... - # end - # end - # - # This way, the result of `can_push_to_branch?` would be cached in - # `RequestStore.store` based on the cache key. If RequestStore is not - # currently active, then it would be stored in a hash saved in an - # instance variable, so the cache logic would be the same. - # Here's another example using customized method level values: - # - # class Commit - # extend Gitlab::Cache::RequestCache - # - # def author - # User.find_by_any_email(author_email.downcase) - # end - # request_cache(:author) { author_email.downcase } - # end - # - # So that we could have different strategies for different methods - # + # See https://docs.gitlab.com/ee/development/utilities.html#requestcache module RequestCache def self.extended(klass) return if klass < self diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb index 43a52b493bb..22310e313ac 100644 --- a/lib/gitlab/checks/commit_check.rb +++ b/lib/gitlab/checks/commit_check.rb @@ -37,7 +37,7 @@ module Gitlab def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do - project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev + project.lfs_enabled? && newrev && oldrev && project.any_lfs_file_locks? end end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index c9c3050cfc2..87af4a90572 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -7,18 +7,10 @@ module Gitlab # Created or deleted branch return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - GitalyClient.migrate(:force_push) do |is_enabled| - if is_enabled - !project - .repository - .gitaly_commit_client - .ancestor?(oldrev, newrev) - else - Gitlab::Git::RevList.new( - project.repository.raw, oldrev: oldrev, newrev: newrev - ).missed_ref.present? - end - end + !project + .repository + .gitaly_commit_client + .ancestor?(oldrev, newrev) end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index d00e5b07f95..222aa06b800 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -4,6 +4,9 @@ module Gitlab class Collection class Item def initialize(key:, value:, public: true, file: false) + raise ArgumentError, "`value` must be of type String, while it was: #{value.class}" unless + value.is_a?(String) || value.nil? + @variable = { key: key, value: value, public: public, file: file } diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d49d055c3f2..4ad106e7b0a 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -188,8 +188,11 @@ module Gitlab end def self.cached_table_exists?(table_name) - # Rails 5 uses data_source_exists? instead of table_exists? - connection.schema_cache.table_exists?(table_name) + if Gitlab.rails5? + connection.schema_cache.data_source_exists?(table_name) + else + connection.schema_cache.table_exists?(table_name) + end end private_class_method :connection diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index 62d4d0a92a6..26ae6966746 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -37,6 +37,7 @@ module Gitlab class Namespace < ActiveRecord::Base include MigrationClasses::Routable self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled belongs_to :parent, class_name: "#{MigrationClasses.name}::Namespace" has_one :route, as: :source diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 2820293ad5c..40bcfa20e7d 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -130,11 +130,13 @@ module Gitlab # Array of Gitlab::Diff::Line objects def diff_lines - @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a + @diff_lines ||= + Gitlab::Diff::Parser.new.parse(raw_diff.each_line, diff_file: self).to_a end def highlighted_diff_lines - @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight + @highlighted_diff_lines ||= + Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted @@ -239,8 +241,33 @@ module Gitlab simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?) end + # This adds the bottom match line to the array if needed. It contains + # the data to load more context lines. + def diff_lines_for_serializer + lines = highlighted_diff_lines + + return if lines.empty? + + last_line = lines.last + + if last_line.new_pos < total_blob_lines(blob) + match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) + lines.push(match_line) + end + + lines + end + private + def total_blob_lines(blob) + @total_lines ||= begin + line_count = blob.lines.size + line_count -= 1 if line_count > 0 && blob.lines.last.blank? + line_count + end + end + # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index a1e904cfef4..2b3ebfbb9ff 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,22 +1,26 @@ module Gitlab module Diff class Line - attr_reader :type, :index, :old_pos, :new_pos + attr_reader :line_code, :type, :index, :old_pos, :new_pos attr_writer :rich_text attr_accessor :text - def initialize(text, type, index, old_pos, new_pos, parent_file: nil) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos @parent_file = parent_file + + # When line code is not provided from cache store we build it + # using the parent_file(Diff::File or Conflict::File). + @line_code = line_code || calculate_line_code end def self.init_from_hash(hash) - new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos]) + new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos], line_code: hash[:line_code]) end def serialize_keys - @serialize_keys ||= %i(text type index old_pos new_pos) + @serialize_keys ||= %i(line_code text type index old_pos new_pos) end def to_hash @@ -62,20 +66,37 @@ module Gitlab end def rich_text - @parent_file.highlight_lines! if @parent_file && !@rich_text + @parent_file.try(:highlight_lines!) if @parent_file && !@rich_text @rich_text end + def meta_positions + return unless meta? + + { + old_pos: old_pos, + new_pos: new_pos + } + end + def as_json(opts = nil) { + line_code: line_code, type: type, old_line: old_line, new_line: new_line, text: text, - rich_text: rich_text || text + rich_text: rich_text || text, + meta_data: meta_positions } end + + private + + def calculate_line_code + @parent_file&.line_code(self) + end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 8302f30a0a2..7ae7ed286ed 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -3,7 +3,7 @@ module Gitlab class Parser include Enumerable - def parse(lines) + def parse(lines, diff_file: nil) return [] if lines.blank? @lines = lines @@ -31,17 +31,17 @@ module Gitlab next if line_old <= 1 && line_new <= 1 # top of file - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 next elsif line[0] == '\\' type = "#{context}-nonewline" - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 else type = identification_type(line) - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 451c9daf761..d512fc58e46 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -2,10 +2,10 @@ module Gitlab class Favicon class << self def main - return appearance_favicon.favicon_main.url if appearance_favicon.exists? - image_name = - if Gitlab::Utils.to_boolean(ENV['CANARY']) + if appearance_favicon.exists? + appearance_favicon.url + elsif Gitlab::Utils.to_boolean(ENV['CANARY']) 'favicon-yellow.png' elsif Rails.env.development? 'favicon-blue.png' @@ -13,7 +13,7 @@ module Gitlab 'favicon.png' end - ActionController::Base.helpers.image_path(image_name) + ActionController::Base.helpers.image_path(image_name, host: host) end def status_overlay(status_name) @@ -22,7 +22,7 @@ module Gitlab "#{status_name}.png" ) - ActionController::Base.helpers.image_path(path) + ActionController::Base.helpers.image_path(path, host: host) end def available_status_names @@ -35,6 +35,16 @@ module Gitlab private + # we only want to create full urls when there's a different asset_host + # configured. + def host + if Gitlab::Application.config.asset_host.nil? || Gitlab::Application.config.asset_host == Gitlab.config.gitlab.base_url + nil + else + Gitlab.config.gitlab.base_url + end + end + def appearance RequestStore.store[:appearance] ||= (Appearance.current || Appearance.new) end diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index f42088f980e..af8270c8db8 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -14,14 +14,21 @@ module Gitlab end def find(query) - by_content = find_by_content(query) + query = Gitlab::Search::Query.new(query) do + filter :filename, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}$/i } + filter :path, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}/i } + filter :extension, matcher: ->(filter, blob) { blob.filename =~ /\.#{filter[:regex_value]}$/i } + end + + by_content = find_by_content(query.term) already_found = Set.new(by_content.map(&:filename)) - by_filename = find_by_filename(query, except: already_found) + by_filename = find_by_filename(query.term, except: already_found) + + files = (by_content + by_filename) + .sort_by(&:filename) - (by_content + by_filename) - .sort_by(&:filename) - .map { |blob| [blob.filename, blob] } + query.filter_results(files).map { |blob| [blob.filename, blob] } end private diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 40b65f6c0da..e25e15f5c80 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -22,24 +22,9 @@ module Gitlab private def load_blame - raw_output = @repo.gitaly_migrate(:blame, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - load_blame_by_gitaly - else - load_blame_by_shelling_out - end - end - - output = encode_utf8(raw_output) - process_raw_blame output - end - - def load_blame_by_gitaly - @repo.gitaly_commit_client.raw_blame(@sha, @path) - end + output = encode_utf8(@repo.gitaly_commit_client.raw_blame(@sha, @path)) - def load_blame_by_shelling_out - @repo.shell_blame(@sha, @path) + process_raw_blame(output) end def process_raw_blame(output) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 156d077a69c..604bb11e712 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -21,13 +21,31 @@ module Gitlab attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary class << self - def find(repository, sha, path) - Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| - if is_enabled - find_by_gitaly(repository, sha, path) - else - find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - end + def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + + path = path.sub(%r{\A/*}, '') + path = '/' if path.empty? + name = File.basename(path) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) + return unless entry + + entry.data = "" if limit == 0 + + case entry.type + when :COMMIT + new(id: entry.oid, name: name, size: 0, data: '', path: path, commit_id: sha) + when :BLOB + new(id: entry.oid, name: name, size: entry.size, data: entry.data.dup, mode: entry.mode.to_s(8), + path: path, commit_id: sha, binary: binary?(entry.data)) end end @@ -56,7 +74,7 @@ module Gitlab repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + find(repository, sha, path, limit: blob_size_limit) end end end @@ -136,85 +154,6 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - return unless path - - path = path.sub(%r{\A/*}, '') - path = '/' if path.empty? - name = File.basename(path) - - # Gitaly will think that setting the limit to 0 means unlimited, while - # the client might only need the metadata and thus set the limit to 0. - # In this method we'll then set the limit to 1, but clear the byte of data - # that we got back so for the outside world it looks like the limit was - # actually 0. - req_limit = limit == 0 ? 1 : limit - - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) - return unless entry - - entry.data = "" if limit == 0 - - case entry.type - when :COMMIT - new( - id: entry.oid, - name: name, - size: 0, - data: '', - path: path, - commit_id: sha - ) - when :BLOB - new( - id: entry.oid, - name: name, - size: entry.size, - data: entry.data.dup, - mode: entry.mode.to_s(8), - path: path, - commit_id: sha, - binary: binary?(entry.data) - ) - end - end - - def find_by_rugged(repository, sha, path, limit:) - return unless path - - # Strip any leading / characters from the path - path = path.sub(%r{\A/*}, '') - - rugged_commit = repository.lookup(sha) - root_tree = rugged_commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) - - return nil unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit.zero? ? '' : blob.content(limit), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) - end - end - rescue Rugged::ReferenceError - nil - end - def rugged_raw(repository, sha, limit:) blob = repository.lookup(sha) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c9806cdb85f..341768752dc 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -381,15 +381,11 @@ module Gitlab # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.diff_from_parent(self, options) - else - rugged_diff_from_parent(options) - end - end + @repository.gitaly_commit_client.diff_from_parent(self, options) end + # Not to be called directly, but right now its used for tests and in old + # migrations def rugged_diff_from_parent(options = {}) options ||= {} break_rewrites = options[:break_rewrites] diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index 8475645971e..5ff15a787f0 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -61,22 +61,15 @@ module Gitlab end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - tags_option = tags ? '--tags' : '--no-tags' - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet) - cmd << '--prune' if prune - cmd << '--force' if force - cmd << tags_option + cmd = fetch_remote_command(name, tags, prune, force) setup_ssh_auth(ssh_key, known_hosts) do |env| - success = run_with_timeout(cmd, timeout, repository_absolute_path, env) - - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| + unless success + logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + end end - - success end end @@ -202,6 +195,14 @@ module Gitlab private + def fetch_remote_command(name, tags, prune, force) + %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| + cmd << '--prune' if prune + cmd << '--force' if force + cmd << (tags ? '--tags' : '--no-tags') + end + end + def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index f3cc388ea41..f0fab1e76a3 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,67 +7,11 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) - @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) - else - git_new_pointers(object_limit, not_in) - end - end + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) end def all_pointers - @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) - else - git_all_pointers - end - end - end - - private - - def git_new_pointers(object_limit, not_in) - @new_pointers ||= begin - rev_list.new_objects(rev_list_params(not_in: not_in)) do |object_ids| - object_ids = object_ids.take(object_limit) if object_limit - - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - end - - def git_all_pointers - params = {} - if rev_list_supports_new_options? - params[:options] = ["--filter=blob:limit=#{Gitlab::Git::Blob::LFS_POINTER_MAX_SIZE}"] - end - - rev_list.all_objects(rev_list_params(params)) do |object_ids| - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - - def rev_list - Gitlab::Git::RevList.new(@repository, newrev: @newrev) - end - - # We're passing the `--in-commit-order` arg to ensure we don't wait - # for git to traverse all commits before returning pointers. - # This is required in order to improve the performance of LFS integrity check - def rev_list_params(params = {}) - params[:options] ||= [] - params[:options] << "--in-commit-order" if rev_list_supports_new_options? - params[:require_path] = true - - params - end - - def rev_list_supports_new_options? - return @option_supported if defined?(@option_supported) - - @option_supported = Gitlab::Git.version >= Gitlab::VersionInfo.parse('2.16.0') + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) end end end diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index ebe46722890..e4743b4db0a 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -7,81 +7,8 @@ module Gitlab end def update(only_branches_matching: []) - @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| - if is_enabled - gitaly_update(only_branches_matching) - else - rugged_update(only_branches_matching) - end - end - end - - private - - def gitaly_update(only_branches_matching) - @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) - end - - def rugged_update(only_branches_matching) - local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) - remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) - - updated_branches = changed_refs(local_branches, remote_branches) - push_branches(updated_branches.keys) if updated_branches.present? - - delete_refs(local_branches, remote_branches) - - local_tags = refs_obj(@repository.tags) - remote_tags = refs_obj(@repository.remote_tags(@ref_name)) - - updated_tags = changed_refs(local_tags, remote_tags) - @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? - - delete_refs(local_tags, remote_tags) - end - - def refs_obj(refs, only_refs_matching: []) - refs.each_with_object({}) do |ref, refs| - next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) - - refs[ref.name] = ref - end - end - - def changed_refs(local_refs, remote_refs) - local_refs.select do |ref_name, ref| - remote_ref = remote_refs[ref_name] - - remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target - end - end - - def push_branches(branches) - default_branch, branches = branches.partition do |branch| - @repository.root_ref == branch - end - - # Push the default branch first so it works fine when remote mirror is empty. - branches.unshift(*default_branch) - - @repository.push_remote_branches(@ref_name, branches) - end - - def delete_refs(local_refs, remote_refs) - refs = refs_to_delete(local_refs, remote_refs) - - @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? - end - - def refs_to_delete(local_refs, remote_refs) - default_branch_id = @repository.commit.id - - remote_refs.select do |remote_ref_name, remote_ref| - next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo - - remote_ref_id = remote_ref.dereferenced_target.try(:id) - - remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + @repository.wrapped_gitaly_errors do + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e883964a090..b3016c1a637 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -120,13 +120,11 @@ module Gitlab # Default branch in the repository def root_ref - @root_ref ||= gitaly_migrate(:root_ref, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.default_branch_name - else - discover_default_branch - end - end + gitaly_ref_client.default_branch_name + rescue GRPC::NotFound => e + raise NoRepository.new(e.message) + rescue GRPC::Unknown => e + raise Gitlab::Git::CommandError.new(e.message) end def rugged @@ -152,23 +150,15 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - gitaly_migrate(:branch_names, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.branch_names - else - branches.map(&:name) - end + wrapped_gitaly_errors do + gitaly_ref_client.branch_names end end # Returns an Array of Branches def branches - gitaly_migrate(:branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.branches - else - branches_filter - end + wrapped_gitaly_errors do + gitaly_ref_client.branches end end @@ -200,12 +190,8 @@ module Gitlab end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.local_branches(sort_by: sort_by) - else - branches_filter(filter: :local, sort_by: sort_by) - end + wrapped_gitaly_errors do + gitaly_ref_client.local_branches(sort_by: sort_by) end end @@ -245,18 +231,6 @@ module Gitlab # This refs by default not visible in project page and not cloned to client side. alias_method :has_visible_content?, :has_local_branches? - def has_local_branches_rugged? - rugged.branches.each(:local).any? do |ref| - begin - ref.name && ref.target # ensures the branch is valid - - true - rescue Rugged::ReferenceError - false - end - end - end - # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -270,12 +244,8 @@ module Gitlab # Returns an Array of tag names def tag_names - gitaly_migrate(:tag_names, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.tag_names - else - rugged.tags.map { |t| t.name } - end + wrapped_gitaly_errors do + gitaly_ref_client.tag_names end end @@ -283,12 +253,8 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags - gitaly_migrate(:tags, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - tags_from_gitaly - else - tags_from_rugged - end + wrapped_gitaly_errors do + gitaly_ref_client.tags end end @@ -364,31 +330,6 @@ module Gitlab end.map(&:name) end - # Discovers the default branch based on the repository's available branches - # - # - If no branches are present, returns nil - # - If one branch is present, returns its name - # - If two or more branches are present, returns current HEAD or master or first branch - def discover_default_branch - names = branch_names - - return if names.empty? - - return names[0] if names.length == 1 - - if rugged_head - extracted_name = Ref.extract_branch_name(rugged_head.name) - - return extracted_name if names.include?(extracted_name) - end - - if names.include?('master') - 'master' - else - names[0] - end - end - def rugged_head rugged.head rescue Rugged::ReferenceError @@ -462,13 +403,7 @@ module Gitlab # Return repo size in megabytes def size - size = gitaly_migrate(:repository_size) do |is_enabled| - if is_enabled - size_by_gitaly - else - size_by_shelling_out - end - end + size = gitaly_repository_client.repository_size (size.to_f / 1024).round(2) end @@ -531,13 +466,21 @@ module Gitlab end def count_commits(options) - count_commits_options = process_count_commits_options(options) + options = process_count_commits_options(options.dup) - gitaly_migrate(:count_commits, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - count_commits_by_gitaly(count_commits_options) + wrapped_gitaly_errors do + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] else - count_commits_by_shelling_out(count_commits_options) + gitaly_commit_client.commit_count(options[:ref], options) end end end @@ -580,32 +523,17 @@ module Gitlab def raw_changes_between(old_rev, new_rev) @raw_changes_between ||= {} - @raw_changes_between[[old_rev, new_rev]] ||= begin - return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA + @raw_changes_between[[old_rev, new_rev]] ||= + begin + return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA - gitaly_migrate(:raw_changes_between) do |is_enabled| - if is_enabled + wrapped_gitaly_errors do gitaly_repository_client.raw_changes_between(old_rev, new_rev) .each_with_object([]) do |msg, arr| msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) } end - else - result = [] - - circuit_breaker.perform do - Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads| - last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) } - - if wait_threads.any? { |waiter| !waiter.value&.success? } - raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}" - end - end - end - - result end end - end rescue ArgumentError => e raise Gitlab::Git::Repository::GitError.new(e) end @@ -621,24 +549,9 @@ module Gitlab end end - # Gitaly note: JV: check gitlab-ee before removing this method. - def rugged_is_ancestor?(ancestor_id, descendant_id) - return false if ancestor_id.nil? || descendant_id.nil? - - rugged_merge_base(ancestor_id, descendant_id) == ancestor_id - rescue Rugged::OdbError - false - end - # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| - if is_enabled - gitaly_commit_client.ancestor?(from, to) - else - rugged_is_ancestor?(from, to) - end - end + gitaly_commit_client.ancestor?(from, to) end def merged_branch_names(branch_names = []) @@ -679,17 +592,7 @@ module Gitlab def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? - gitaly_migrate(:find_ref_name) do |is_enabled| - if is_enabled - gitaly_ref_client.find_ref_name(sha, ref_path) - else - args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - run_git(args).first.split.last - end - end + gitaly_ref_client.find_ref_name(sha, ref_path) end # Get refs hash which key is is the commit id @@ -735,15 +638,9 @@ module Gitlab end # Return total commits count accessible from passed ref - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) - gitaly_migrate(:commit_count, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_commit_client.commit_count(ref) - else - rugged_commit_count(ref) - end + wrapped_gitaly_errors do + gitaly_commit_client.commit_count(ref) end end @@ -1018,13 +915,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - gitaly_migrate(:ls_files) do |is_enabled| - if is_enabled - gitaly_ls_files(ref) - else - git_ls_files(ref) - end - end + gitaly_commit_client.ls_files(ref) end # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 @@ -1043,21 +934,7 @@ module Gitlab def info_attributes return @info_attributes if @info_attributes - content = - gitaly_migrate(:get_info_attributes, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.info_attributes - else - attributes_path = File.join(File.expand_path(path), 'info', 'attributes') - - if File.exist?(attributes_path) - File.read(attributes_path) - else - "" - end - end - end - + content = gitaly_repository_client.info_attributes @info_attributes = AttributesParser.new(content) end @@ -1086,45 +963,14 @@ module Gitlab end def languages(ref = nil) - gitaly_migrate(:commit_languages, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_commit_client.languages(ref) - else - ref ||= rugged.head.target_id - languages = Linguist::Repository.new(rugged, ref).languages - total = languages.map(&:last).sum - - languages = languages.map do |language| - name, share = language - color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" - { - value: (share.to_f * 100 / total).round(2), - label: name, - color: color, - highlight: color - } - end - - languages.sort do |x, y| - y[:value] <=> x[:value] - end - end + wrapped_gitaly_errors do + gitaly_commit_client.languages(ref) end end def license_short_name - gitaly_migrate(:license_short_name, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.license_short_name - else - begin - # The licensee gem creates a Rugged object from the path: - # https://github.com/benbalter/licensee/blob/v8.7.0/lib/licensee/projects/git_project.rb - Licensee.license(path).try(:key) - rescue Rugged::Error - end - end + wrapped_gitaly_errors do + gitaly_repository_client.license_short_name end end @@ -1181,18 +1027,18 @@ module Gitlab end def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - with_repo_branch_commit(source_repository, source_branch_name) do |commit| - break unless commit - - Gitlab::Git::Compare.new( - self, - target_branch_name, - commit.sha, - straight: straight - ) - end - end + tmp_ref = "refs/tmp/#{SecureRandom.hex}" + + return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref) + + Gitlab::Git::Compare.new( + self, + target_branch_name, + tmp_ref, + straight: straight + ) + ensure + delete_refs(tmp_ref) end def write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1276,16 +1122,7 @@ module Gitlab end def create_from_bundle(bundle_path) - gitaly_migrate(:create_repo_from_bundle) do |is_enabled| - if is_enabled - gitaly_repository_client.create_from_bundle(bundle_path) - else - run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil) - self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) - end - end - - true + gitaly_repository_client.create_from_bundle(bundle_path) end def create_from_snapshot(url, auth) @@ -1311,12 +1148,8 @@ module Gitlab end def rebase_in_progress?(rebase_id) - gitaly_migrate(:rebase_in_progress) do |is_enabled| - if is_enabled - gitaly_repository_client.rebase_in_progress?(rebase_id) - else - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.rebase_in_progress?(rebase_id) end end @@ -1332,12 +1165,8 @@ module Gitlab end def squash_in_progress?(squash_id) - gitaly_migrate(:squash_in_progress, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.squash_in_progress?(squash_id) - else - fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.squash_in_progress?(squash_id) end end @@ -1394,16 +1223,10 @@ module Gitlab return unless full_path.present? # This guard avoids Gitaly log/error spam - unless exists? - raise NoRepository, 'repository does not exist' - end + raise NoRepository, 'repository does not exist' unless exists? - gitaly_migrate(:write_config) do |is_enabled| - if is_enabled - gitaly_repository_client.write_config(full_path: full_path) - else - rugged_write_config(full_path: full_path) - end + wrapped_gitaly_errors do + gitaly_repository_client.write_config(full_path: full_path) end end @@ -1453,6 +1276,16 @@ module Gitlab raise CommandError.new(e) end + def wrapped_gitaly_errors(&block) + yield block + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + def clean_stale_repository_files gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| gitaly_repository_client.cleanup if is_enabled && exists? @@ -1492,12 +1325,10 @@ module Gitlab end def can_be_merged?(source_sha, target_branch) - gitaly_migrate(:can_be_merged) do |is_enabled| - if is_enabled - gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target) - else - rugged_can_be_merged?(source_sha, target_branch) - end + if target_sha = find_branch(target_branch, true)&.target + !gitaly_conflicts_client(source_sha, target_sha).conflicts? + else + false end end @@ -1563,10 +1394,6 @@ module Gitlab run_git!(args, lazy_block: block) end - def missed_ref(oldrev, newrev) - run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) - end - def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) base_args = %w(worktree add --detach) @@ -1606,12 +1433,8 @@ module Gitlab private def uncached_has_local_branches? - gitaly_migrate(:has_local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.has_local_branches? - else - has_local_branches_rugged? - end + wrapped_gitaly_errors do + gitaly_repository_client.has_local_branches? end end @@ -1674,21 +1497,6 @@ module Gitlab end end - # This function is duplicated in Gitaly-Go, don't change it! - # https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - def fresh_worktree?(path) - File.exist?(path) && !clean_stuck_worktree(path) - end - - # This function is duplicated in Gitaly-Go, don't change it! - # https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - def clean_stuck_worktree(path) - return false unless File.mtime(path) < 15.minutes.ago - - FileUtils.rm_rf(path) - true - end - # Adding a worktree means checking out the repository. For large repos, # this can be very expensive, so set up sparse checkout for the worktree # to only check out the files we're interested in. @@ -1731,20 +1539,6 @@ module Gitlab } end - # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. - def branches_filter(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - - sort_branches(branches, sort_by) - end - def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} @@ -1956,137 +1750,11 @@ module Gitlab end end - def tags_from_rugged - rugged.references.each("refs/tags/*").map do |ref| - message = nil - - if ref.target.is_a?(Rugged::Tag::Annotation) - tag_message = ref.target.message - - if tag_message.respond_to?(:chomp) - message = tag_message.chomp - end - end - - target_commit = Gitlab::Git::Commit.find(self, ref.target) - Gitlab::Git::Tag.new(self, { - name: ref.name, - target: ref.target, - target_commit: target_commit, - message: message - }) - end.sort_by(&:name) - end - def last_commit_for_path_by_rugged(sha, path) sha = last_commit_id_for_path_by_shelling_out(sha, path) commit(sha) end - def tags_from_gitaly - gitaly_ref_client.tags - end - - def size_by_shelling_out - popen(%w(du -sk), path).first.strip.to_i - end - - def size_by_gitaly - gitaly_repository_client.repository_size - end - - def count_commits_by_gitaly(options) - if options[:left_right] - from = options[:from] - to = options[:to] - - right_count = gitaly_commit_client - .commit_count("#{from}..#{to}", options) - left_count = gitaly_commit_client - .commit_count("#{to}..#{from}", options) - - [left_count, right_count] - else - gitaly_commit_client.commit_count(options[:ref], options) - end - end - - def count_commits_by_shelling_out(options) - cmd = count_commits_shelling_command(options) - - raw_output, _status = run_git(cmd) - - process_count_commits_raw_output(raw_output, options) - end - - def count_commits_shelling_command(options) - cmd = %w[rev-list] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << "--max-count=#{options[:max_count]}" if options[:max_count] - cmd << "--left-right" if options[:left_right] - cmd << '--count' - - cmd << if options[:all] - '--all' - elsif options[:ref] - options[:ref] - else - raise ArgumentError, "Please specify a valid ref or set the 'all' attribute to true" - end - - cmd += %W[-- #{options[:path]}] if options[:path].present? - cmd - end - - def process_count_commits_raw_output(raw_output, options) - if options[:left_right] - result = raw_output.scan(/\d+/).map(&:to_i) - - if result.sum != options[:max_count] - result - else # Reaching max count, right is not accurate - right_option = - process_count_commits_options(options - .except(:left_right, :from, :to) - .merge(ref: options[:to])) - - right = count_commits_by_shelling_out(right_option) - - [result.first, right] # left should be accurate in the first call - end - else - raw_output.to_i - end - end - - def gitaly_ls_files(ref) - gitaly_commit_client.ls_files(ref) - end - - def git_ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) - raw_output, _status = run_git(cmd) - - lines = raw_output.split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type - end - - lines.compact - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -2439,14 +2107,6 @@ module Gitlab run_git(['fetch', remote_name], env: env).last.zero? end - def gitaly_can_be_merged?(their_commit, our_commit) - !gitaly_conflicts_client(our_commit, their_commit).conflicts? - end - - def rugged_can_be_merged?(their_commit, our_commit) - !rugged.merge_commits(our_commit, their_commit).conflicts? - end - def gitlab_projects_error raise CommandError, @gitlab_projects.output end @@ -2486,16 +2146,6 @@ module Gitlab nil end - def rugged_commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count - rescue Rugged::ReferenceError - 0 - end - def rev_list_param(spec) spec == :all ? ['--all'] : spec end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 4e661eceffb..5fdad077eea 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. - module Gitlab module Git class RevList @@ -45,13 +43,6 @@ module Gitlab &lazy_block) end - # This methods returns an array of missed references - # - # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. - def missed_ref - repository.missed_ref(oldrev, newrev).split("\n") - end - private def execute(args) diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb index 11184ca3457..1e14e8b652a 100644 --- a/lib/gitlab/git/version.rb +++ b/lib/gitlab/git/version.rb @@ -4,7 +4,7 @@ module Gitlab extend Gitlab::Git::Popen def self.git_version - Gitlab::VersionInfo.parse(popen(%W(#{Gitlab.config.git.bin_path} --version), nil).first) + Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 1ab8c4e0229..8ee46b59830 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -27,63 +27,38 @@ module Gitlab end def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| - if is_enabled - gitaly_write_page(name, format, content, commit_details) - else - gollum_write_page(name, format, content, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_write_page(name, format, content, commit_details) end end def delete_page(page_path, commit_details) - @repository.gitaly_migrate(:wiki_delete_page) do |is_enabled| - if is_enabled - gitaly_delete_page(page_path, commit_details) - else - gollum_delete_page(page_path, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_delete_page(page_path, commit_details) end end def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| - if is_enabled - gitaly_update_page(page_path, title, format, content, commit_details) - else - gollum_update_page(page_path, title, format, content, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_update_page(page_path, title, format, content, commit_details) end end def pages(limit: nil) - @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| - if is_enabled - gitaly_get_all_pages - else - gollum_get_all_pages(limit: limit) - end + @repository.wrapped_gitaly_errors do + gitaly_get_all_pages end end def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_find_page(title: title, version: version, dir: dir) - else - gollum_find_page(title: title, version: version, dir: dir) - end + @repository.wrapped_gitaly_errors do + gitaly_find_page(title: title, version: version, dir: dir) end end def file(name, version) - @repository.gitaly_migrate(:wiki_find_file) do |is_enabled| - if is_enabled - gitaly_find_file(name, version) - else - gollum_find_file(name, version) - end + @repository.wrapped_gitaly_errors do + gitaly_find_file(name, version) end end @@ -92,24 +67,15 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| - if is_enabled - versions = gitaly_wiki_client.page_versions(page_path, options) - - # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 - # per page, but also fetches 20 if `limit` or `per_page` < 20. - # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page - versions[0..slice_bound] - else - current_page = gollum_page_by_path(page_path) - - commits_from_page(current_page, options).map do |gitlab_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) - Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) - end - end + versions = @repository.wrapped_gitaly_errors do + gitaly_wiki_client.page_versions(page_path, options) end + + # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 + # per page, but also fetches 20 if `limit` or `per_page` < 20. + # Slicing returns an array with the expected number of items. + slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + versions[0..slice_bound] end def count_page_versions(page_path) @@ -131,46 +97,13 @@ module Gitlab def page_formatted_data(title:, dir: nil, version: nil) version = version&.id - @repository.gitaly_migrate(:wiki_page_formatted_data, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) - else - # We don't use #page because if wiki_find_page feature is enabled, we would - # get a page without formatted_data. - gollum_find_page(title: title, dir: dir, version: version)&.formatted_data - end + @repository.wrapped_gitaly_errors do + gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) end end - def gollum_wiki - @gollum_wiki ||= Gollum::Wiki.new(@repository.path) - end - private - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def commits_from_page(gollum_page, options = {}) - unless options[:limit] - options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page - options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i - end - - @repository.log(ref: gollum_page.last_version.id, - path: gollum_page.path, - limit: options[:limit], - offset: options[:offset]) - end - - def gollum_page_by_path(page_path) - page_name = Gollum::Page.canonicalize_filename(page_path) - page_dir = File.split(page_path).first - - gollum_wiki.paged(page_name, page_dir) - end - def new_page(gollum_page) Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) end @@ -199,65 +132,6 @@ module Gitlab @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end - def gollum_write_page(name, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - filename = File.basename(name) - dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir - - gollum_wiki.write_page(filename, format, content, { committer: committer }, dir) - end - rescue Gollum::DuplicatePageError => e - raise Gitlab::Git::Wiki::DuplicatePageError, e.message - end - - def gollum_delete_page(page_path, commit_details) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - gollum_wiki.delete_page(gollum_page_by_path(page_path), committer: committer) - end - end - - def gollum_update_page(page_path, title, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - page = gollum_page_by_path(page_path) - # Instead of performing two renames if the title has changed, - # the update_page will only update the format and content and - # the rename_page will do anything related to moving/renaming - gollum_wiki.update_page(page, page.name, format, content, committer: committer) - gollum_wiki.rename_page(page, title, committer: committer) - end - end - - def gollum_find_page(title:, version: nil, dir: nil) - if version - version = Gitlab::Git::Commit.find(@repository, version).id - end - - gollum_page = gollum_wiki.page(title, version, dir) - return unless gollum_page - - new_page(gollum_page) - end - - def gollum_find_file(name, version) - version ||= self.class.default_ref - gollum_file = gollum_wiki.file(name, version) - return unless gollum_file - - Gitlab::Git::WikiFile.new(gollum_file) - end - - def gollum_get_all_pages(limit: nil) - gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } - end - def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 36e9adf27da..620362b52a9 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -33,11 +33,6 @@ module Gitlab MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze - # We have a mechanism to let GitLab automatically opt in to all Gitaly - # features. We want to be able to exclude some features from automatic - # opt-in. That is what EXPLICIT_OPT_IN_REQUIRED is for. - EXPLICIT_OPT_IN_REQUIRED = [Gitlab::GitalyClient::StorageSettings::DISK_ACCESS_DENIED_FLAG].freeze - MUTEX = Mutex.new class << self @@ -249,7 +244,7 @@ module Gitlab when MigrationStatus::OPT_OUT true when MigrationStatus::OPT_IN - opt_into_all_features? && !EXPLICIT_OPT_IN_REQUIRED.include?(feature_name) + opt_into_all_features? && !explicit_opt_in_required.include?(feature_name) else false end @@ -259,6 +254,13 @@ module Gitlab false end + # We have a mechanism to let GitLab automatically opt in to all Gitaly + # features. We want to be able to exclude some features from automatic + # opt-in. This function has an override in EE. + def self.explicit_opt_in_required + [] + end + # opt_into_all_features? returns true when the current environment # is one in which we opt into features automatically def self.opt_into_all_features? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index a4cc64de80d..7f2e6441f16 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -179,6 +179,8 @@ module Gitlab end def list_commits_by_oid(oids) + return [] if oids.empty? + request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids) response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 4340f779e53..ca986434221 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -196,20 +196,21 @@ module Gitlab end def create_bundle(save_path) - request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) - response = GitalyClient.call( - @storage, - :repository_service, + gitaly_fetch_stream_to_file( + save_path, :create_bundle, - request, - timeout: GitalyClient.default_timeout + Gitaly::CreateBundleRequest, + GitalyClient.default_timeout ) + end - File.open(save_path, 'wb') do |f| - response.each do |message| - f.write(message.data) - end - end + def backup_custom_hooks(save_path) + gitaly_fetch_stream_to_file( + save_path, + :backup_custom_hooks, + Gitaly::BackupCustomHooksRequest, + GitalyClient.default_timeout + ) end def create_from_bundle(bundle_path) @@ -309,6 +310,25 @@ module Gitlab private + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + rpc_name, + request, + timeout: timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + # If the file is empty means that we recieved an empty stream, we delete the file + FileUtils.rm(save_path) if File.zero?(save_path) + end + def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) enum = Enumerator.new do |y| diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index b02b123c98e..a77ac1e4fa6 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -15,6 +15,15 @@ module Gitlab true end + # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore + # the visibility of prepended modules. See + # https://github.com/rspec/rspec-mocks/issues/1231 for more details. + if Rails.env.test? + def self.requires_ci_cd_setup? + raise NotImplementedError + end + end + def initialize(project) @project = project end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index e27e16ddaf6..08495c0a59e 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -17,7 +17,7 @@ module Gitlab def check catch_timeout 10.seconds do if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping') + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s else ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index fcbf266b80b..050fe7a5173 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -1,5 +1,6 @@ module Gitlab module HealthChecks + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1218 class FsShardsCheck extend BaseAbstractCheck RANDOM_STRING = SecureRandom.hex(1000).freeze diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3772ef11c7f..343487bc361 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -21,7 +21,8 @@ module Gitlab 'nl_NL' => 'Nederlands', 'tr_TR' => 'Türkçe', 'id_ID' => 'Bahasa Indonesia', - 'fil_PH' => 'Filipino' + 'fil_PH' => 'Filipino', + 'pl_PL' => 'Polski' }.freeze def available_locales diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb index 35d57459a3d..36fc1bcdcb7 100644 --- a/lib/gitlab/i18n/metadata_entry.rb +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -3,16 +3,25 @@ module Gitlab class MetadataEntry attr_reader :entry_data + # Avoid testing too many plurals if `nplurals` was incorrectly set. + # Based on info on https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html + # which mentions special cases for numbers ending in 2 digits + MAX_FORMS_TO_TEST = 101 + def initialize(entry_data) @entry_data = entry_data end - def expected_plurals + def expected_forms return nil unless plural_information plural_information['nplurals'].to_i end + def forms_to_test + @forms_to_test ||= [expected_forms, MAX_FORMS_TO_TEST].compact.min + end + private def plural_information diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 7d3ff8c7f58..d8e7269a2c2 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -1,6 +1,8 @@ module Gitlab module I18n class PoLinter + include Gitlab::Utils::StrongMemoize + attr_reader :po_path, :translation_entries, :metadata_entry, :locale VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze @@ -34,7 +36,7 @@ module Gitlab end @translation_entries = entries.map do |entry_data| - Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals) + Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms) end nil @@ -48,7 +50,7 @@ module Gitlab translation_entries.each do |entry| errors_for_entry = validate_entry(entry) - errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? + errors[entry.msgid] = errors_for_entry if errors_for_entry.any? end errors @@ -62,6 +64,7 @@ module Gitlab validate_newlines(errors, entry) validate_number_of_plurals(errors, entry) validate_unescaped_chars(errors, entry) + validate_translation(errors, entry) errors end @@ -81,35 +84,39 @@ module Gitlab end def validate_number_of_plurals(errors, entry) - return unless metadata_entry&.expected_plurals + return unless metadata_entry&.expected_forms return unless entry.translated? - if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals - errors << "should have #{metadata_entry.expected_plurals} "\ - "#{'translations'.pluralize(metadata_entry.expected_plurals)}" + if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_forms + errors << "should have #{metadata_entry.expected_forms} "\ + "#{'translations'.pluralize(metadata_entry.expected_forms)}" end end def validate_newlines(errors, entry) - if entry.msgid_contains_newlines? + if entry.msgid_has_multiple_lines? errors << 'is defined over multiple lines, this breaks some tooling.' end - if entry.plural_id_contains_newlines? + if entry.plural_id_has_multiple_lines? errors << 'plural is defined over multiple lines, this breaks some tooling.' end - if entry.translations_contain_newlines? + if entry.translations_have_multiple_lines? errors << 'has translations defined over multiple lines, this breaks some tooling.' end end def validate_variables(errors, entry) if entry.has_singular_translation? + validate_variables_in_message(errors, entry.msgid, entry.msgid) + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) end if entry.has_plural? + validate_variables_in_message(errors, entry.plural_id, entry.plural_id) + entry.plural_translations.each do |translation| validate_variables_in_message(errors, entry.plural_id, translation) end @@ -117,41 +124,98 @@ module Gitlab end def validate_variables_in_message(errors, message_id, message_translation) - message_id = join_message(message_id) required_variables = message_id.scan(VARIABLE_REGEX) validate_unnamed_variables(errors, required_variables) - validate_translation(errors, message_id, required_variables) validate_variable_usage(errors, message_translation, required_variables) end - def validate_translation(errors, message_id, used_variables) + def validate_translation(errors, entry) + Gitlab::I18n.with_locale(locale) do + if entry.has_plural? + translate_plural(entry) + else + translate_singular(entry) + end + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale}: #{e.message}" + end + + def translate_singular(entry) + used_variables = entry.msgid.scan(VARIABLE_REGEX) variables = fill_in_variables(used_variables) - begin - Gitlab::I18n.with_locale(locale) do - translated = if message_id.include?('|') - FastGettext::Translation.s_(message_id) - else - FastGettext::Translation._(message_id) - end + translation = if entry.msgid.include?('|') + FastGettext::Translation.s_(entry.msgid) + else + FastGettext::Translation._(entry.msgid) + end - translated % variables + translation % variables if used_variables.any? + end + + def translate_plural(entry) + used_variables = entry.plural_id.scan(VARIABLE_REGEX) + variables = fill_in_variables(used_variables) + + numbers_covering_all_plurals.map do |number| + translation = FastGettext::Translation.n_(entry.msgid, entry.plural_id, number) + + translation % variables if used_variables.any? + end + end + + def numbers_covering_all_plurals + @numbers_covering_all_plurals ||= calculate_numbers_covering_all_plurals + end + + def calculate_numbers_covering_all_plurals + required_numbers = [] + discovered_indexes = [] + counter = 0 + + while discovered_indexes.size < metadata_entry.forms_to_test && counter < Gitlab::I18n::MetadataEntry::MAX_FORMS_TO_TEST + index_for_count = index_for_pluralization(counter) + + unless discovered_indexes.include?(index_for_count) + discovered_indexes << index_for_count + required_numbers << counter end - # `sprintf` could raise an `ArgumentError` when invalid passing something - # other than a Hash when using named variables - # - # `sprintf` could raise `TypeError` when passing a wrong type when using - # unnamed variables - # - # FastGettext::Translation could raise `RuntimeError` (raised as a string), - # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` - # - # `FastGettext::Translation` could raise `ArgumentError` as subclassess - # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` - rescue ArgumentError, TypeError, RuntimeError => e - errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + counter += 1 + end + + required_numbers + end + + def index_for_pluralization(counter) + # This calls the C function that defines the pluralization rule, it can + # return a boolean (`false` represents 0, `true` represents 1) or an integer + # that specifies the plural form to be used for the given number + pluralization_result = Gitlab::I18n.with_locale(locale) do + FastGettext.pluralisation_rule.call(counter) + end + + case pluralization_result + when false + 0 + when true + 1 + else + pluralization_result end end @@ -172,14 +236,18 @@ module Gitlab end def validate_unnamed_variables(errors, variables) - if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) } + + if unnamed_variables.any? && named_variables.any? + errors << 'is combining named variables with unnamed variables' + end + + if unnamed_variables.size > 1 errors << 'is combining multiple unnamed variables' end end def validate_variable_usage(errors, translation, required_variables) - translation = join_message(translation) - # We don't need to validate when the message is empty. # In this case we fall back to the default, which has all the the # required variables. @@ -205,10 +273,6 @@ module Gitlab def validate_flags(errors, entry) errors << "is marked #{entry.flag}" if entry.flag end - - def join_message(message) - Array(message).join - end end end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index e6c95afca7e..54adb98f42d 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -11,11 +11,11 @@ module Gitlab end def msgid - entry_data[:msgid] + @msgid ||= Array(entry_data[:msgid]).join end def plural_id - entry_data[:msgid_plural] + @plural_id ||= Array(entry_data[:msgid_plural]).join end def has_plural? @@ -23,12 +23,11 @@ module Gitlab end def singular_translation - all_translations.first if has_singular_translation? + all_translations.first.to_s if has_singular_translation? end def all_translations - @all_translations ||= entry_data.fetch_values(*translation_keys) - .reject(&:empty?) + @all_translations ||= translation_entries.map { |translation| Array(translation).join } end def translated? @@ -54,16 +53,16 @@ module Gitlab nplurals > 1 || !has_plural? end - def msgid_contains_newlines? - msgid.is_a?(Array) + def msgid_has_multiple_lines? + entry_data[:msgid].is_a?(Array) end - def plural_id_contains_newlines? - plural_id.is_a?(Array) + def plural_id_has_multiple_lines? + entry_data[:msgid_plural].is_a?(Array) end - def translations_contain_newlines? - all_translations.any? { |translation| translation.is_a?(Array) } + def translations_have_multiple_lines? + translation_entries.any? { |translation| translation.is_a?(Array) } end def msgid_contains_unescaped_chars? @@ -84,6 +83,11 @@ module Gitlab private + def translation_entries + @translation_entries ||= entry_data.fetch_values(*translation_keys) + .reject(&:empty?) + end + def translation_keys @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ } end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 30af3e97b4a..d2133a6d65b 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -2,11 +2,12 @@ module Gitlab module Kubernetes module Helm class InstallCommand < BaseCommand - attr_reader :name, :chart, :repository, :values + attr_reader :name, :chart, :version, :repository, :values - def initialize(name, chart:, values:, repository: nil) + def initialize(name, chart:, values:, version: nil, repository: nil) @name = name @chart = chart + @version = version @values = values @repository = repository end @@ -39,9 +40,13 @@ module Gitlab def script_command <<~HEREDOC - helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null + helm install #{chart} --name #{name}#{optional_version_flag} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null HEREDOC end + + def optional_version_flag + " --version #{version}" if version + end end end end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5a0f7f28fc8..ad97632e4eb 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -16,12 +16,6 @@ module Gitlab @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) - - if Gitlab::Metrics.mri? - require 'allocations' - - Allocations.start - end end def sample diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4e1ea62351f..7b2b3bedf04 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -20,39 +20,29 @@ module Gitlab {} end - def initialize(interval) - super(interval) - - if Metrics.mri? - require 'allocations' - - Allocations.start - end - end - def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) - metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) + metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) + metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum) - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum) + metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end def sample start_time = System.monotonic_time - sample_gc - metrics[:memory_usage].set(labels, System.memory_usage) - metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) + metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) + + sample_gc - metrics[:sampler_duration].observe(labels.merge(worker_label), System.monotonic_time - start_time) + metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) ensure GC::Profiler.clear end @@ -60,11 +50,13 @@ module Gitlab private def sample_gc - metrics[:total_time].set(labels, GC::Profiler.total_time * 1000) - + # Collect generic GC stats. GC.stat.each do |key, value| metrics[key].set(labels, value) end + + # Collect the GC time since last sample in float seconds. + metrics[:total_time].increment(labels, GC::Profiler.total_time) end def worker_label diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 4b3e8d0a6a0..38f119cf06d 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -20,7 +20,7 @@ module Gitlab define_histogram :gitlab_sql_duration_seconds do docstring 'SQL time' base_labels Transaction::BASE_LABELS - buckets [0.001, 0.01, 0.1, 1.0, 10.0] + buckets [0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end def current_transaction diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index f3e48083c19..9f903e96585 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -140,7 +140,7 @@ module Gitlab define_histogram :gitlab_transaction_duration_seconds do docstring 'Transaction duration' base_labels BASE_LABELS - buckets [0.001, 0.01, 0.1, 1.0, 10.0] + buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end define_histogram :gitlab_transaction_allocated_memory_bytes do diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 3799aaebf1c..723ca576aab 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -3,6 +3,7 @@ module Gitlab class WebTransaction < Transaction CONTROLLER_KEY = 'action_controller.instance'.freeze ENDPOINT_KEY = 'api.endpoint'.freeze + ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip]) def initialize(env) super() @@ -32,9 +33,13 @@ module Gitlab # Devise exposes a method called "request_format" that does the below. # However, this method is not available to all controllers (e.g. certain # Doorkeeper controllers). As such we use the underlying code directly. - suffix = controller.request.format.try(:ref) + suffix = controller.request.format.try(:ref).to_s - if suffix && suffix != :html + # Sometimes the request format is set to silly data such as + # "application/xrds+xml" or actual URLs. To prevent such values from + # increasing the cardinality of our metrics, we limit the number of + # possible suffixes. + if suffix && ALLOWED_SUFFIXES.include?(suffix) action += ".#{suffix}" end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index e5191f5c7f9..61653044433 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -30,6 +30,7 @@ module Gitlab dashboard deploy.html explore + favicon.ico favicon.png files groups diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 18540e64d4c..ecff6ab5d5e 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -11,6 +11,7 @@ module Gitlab lib/gitlab/etag_caching/ lib/gitlab/metrics/ lib/gitlab/middleware/ + ee/lib/gitlab/middleware/ lib/gitlab/performance_bar/ lib/gitlab/request_profiler/ lib/gitlab/profiler.rb @@ -98,11 +99,7 @@ module Gitlab super - backtrace = Rails.backtrace_cleaner.clean(caller) - - backtrace.each do |caller_line| - next if caller_line.match(Regexp.union(IGNORE_BACKTRACES)) - + Gitlab::Profiler.clean_backtrace(caller).each do |caller_line| stripped_caller_line = caller_line.sub("#{Rails.root}/", '') super(" ↳ #{stripped_caller_line}") @@ -112,6 +109,12 @@ module Gitlab end end + def self.clean_backtrace(backtrace) + Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| + line.match(Regexp.union(IGNORE_BACKTRACES)) + end + end + def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging original_activerecord_logger = ActiveRecord::Base.logger diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 075ff91700c..30c6806b68e 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -39,7 +39,7 @@ module Gitlab content.delete!("\r") content.gsub!(commands_regex) do if $~[:cmd] - commands << [$~[:cmd], $~[:arg]].reject(&:blank?) + commands << [$~[:cmd].downcase, $~[:arg]].reject(&:blank?) '' else $~[0] @@ -102,14 +102,14 @@ module Gitlab # /close ^\/ - (?<cmd>#{Regexp.union(names)}) + (?<cmd>#{Regexp.new(Regexp.union(names).source, Regexp::IGNORECASE)}) (?: [ ] (?<arg>[^\n]*) )? (?:\n|$) ) - }mx + }mix end def perform_substitutions(content, commands) @@ -120,7 +120,7 @@ module Gitlab end substitution_definitions.each do |substitution| - match_data = substitution.match(content) + match_data = substitution.match(content.downcase) if match_data command = [substitution.name.to_s] command << match_data[1] unless match_data[1].empty? diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index 032c49ed159..688056e5d73 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -15,7 +15,7 @@ module Gitlab return unless content all_names.each do |a_name| - content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1')) + content.gsub!(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1')) end content end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index ccfe0d6bed3..a502ad8a541 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -5,7 +5,7 @@ module Gitlab module RequestForgeryProtection class Controller < ActionController::Base - protect_from_forgery with: :exception + protect_from_forgery with: :exception, prepend: true rescue_from ActionController::InvalidAuthenticityToken do |e| logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb new file mode 100644 index 00000000000..23595f23f01 --- /dev/null +++ b/lib/gitlab/search/parsed_query.rb @@ -0,0 +1,23 @@ +module Gitlab + module Search + class ParsedQuery + attr_reader :term, :filters + + def initialize(term, filters) + @term = term + @filters = filters + end + + def filter_results(results) + filters = @filters.reject { |filter| filter[:matcher].nil? } + return unless filters + + results.select do |result| + filters.all? do |filter| + filter[:matcher].call(filter, result) + end + end + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb new file mode 100644 index 00000000000..8583bce7792 --- /dev/null +++ b/lib/gitlab/search/query.rb @@ -0,0 +1,55 @@ +module Gitlab + module Search + class Query < SimpleDelegator + def initialize(query, filter_opts = {}, &block) + @raw_query = query.dup + @filters = [] + @filter_options = { default_parser: :downcase.to_proc }.merge(filter_opts) + + self.instance_eval(&block) if block_given? + + @query = Gitlab::Search::ParsedQuery.new(*extract_filters) + # set the ParsedQuery as our default delegator thanks to SimpleDelegator + super(@query) + end + + private + + def filter(name, **attributes) + filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes) + + @filters << filter + end + + def filter_options(**options) + @filter_options.merge!(options) + end + + def extract_filters + fragments = [] + + filters = @filters.each_with_object([]) do |filter, parsed_filters| + match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ } + next unless match + + input = match.split(':')[1..-1].join + next if input.empty? + + filter[:value] = parse_filter(filter, input) + filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') + fragments << match + + parsed_filters << filter + end + + query = (@raw_query.split - fragments).join(' ') + + [query, filters] + end + + def parse_filter(filter, input) + filter[:parser].call(input) + end + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 4a87f43597e..b2d75aac1d0 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -24,6 +24,7 @@ module Gitlab address = val['gitaly_address'] end + # https://gitlab.com/gitlab-org/gitaly/issues/1238 Gitlab::GitalyClient::StorageSettings.allow_disk_access do storages << { name: key, path: val.legacy_disk_path } end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 4b8aae4f5a2..5cedd9e84c2 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,5 +1,4 @@ -# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. -# SSH key operations are not part of Gitaly so will never be migrated. +# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated. require 'securerandom' @@ -153,8 +152,6 @@ module Gitlab # # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) return false if path.empty? || new_path.empty? @@ -169,19 +166,11 @@ module Gitlab # # Ex. # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) forked_from_relative_path = "#{forked_from_disk_path}.git" fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"] - gitaly_migrate(:fork_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) - else - gitlab_projects(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) - end - end + GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) end # Removes a repository from file system, using rm_diretory which is an alias @@ -193,8 +182,6 @@ module Gitlab # # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) return false if name.empty? diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 824e2d7251f..e64033b0dba 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -26,6 +26,8 @@ module Gitlab project_snippet_url(object.project, object) when Snippet snippet_url(object) + when Milestone + milestone_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 59a222b086c..dff0c97eeb4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -24,7 +24,6 @@ module Gitlab installation_type: Gitlab::INSTALLATION_TYPE, active_user_count: User.active.count, recorded_at: Time.now, - mattermost_enabled: Gitlab.config.mattermost.enabled, edition: 'CE' } @@ -91,13 +90,14 @@ module Gitlab def features_usage_data_ce { - signup: Gitlab::CurrentSettings.allow_signup?, - ldap: Gitlab.config.ldap.enabled, - gravatar: Gitlab::CurrentSettings.gravatar_enabled?, - omniauth: Gitlab.config.omniauth.enabled, - reply_by_email: Gitlab::IncomingEmail.enabled?, - container_registry: Gitlab.config.registry.enabled, - gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled + container_registry_enabled: Gitlab.config.registry.enabled, + gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled, + gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?, + ldap_enabled: Gitlab.config.ldap.enabled, + mattermost_enabled: Gitlab.config.mattermost.enabled, + omniauth_enabled: Gitlab.config.omniauth.enabled, + reply_by_email_enabled: Gitlab::IncomingEmail.enabled?, + signup_enabled: Gitlab::CurrentSettings.allow_signup? } end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb index 1ef369a4b67..167ba1b3149 100644 --- a/lib/gitlab/verify/batch_verifier.rb +++ b/lib/gitlab/verify/batch_verifier.rb @@ -7,13 +7,15 @@ module Gitlab @batch_size = batch_size @start = start @finish = finish + + fix_google_api_logger end # Yields a Range of IDs and a Hash of failed verifications (object => error) def run_batches(&blk) - relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches - range = relation.first.id..relation.last.id - failures = run_batch(relation) + all_relation.in_batches(of: batch_size, start: start, finish: finish) do |batch| # rubocop: disable Cop/InBatches + range = batch.first.id..batch.last.id + failures = run_batch_for(batch) yield(range, failures) end @@ -29,24 +31,56 @@ module Gitlab private - def run_batch(relation) - relation.map { |upload| verify(upload) }.compact.to_h + def run_batch_for(batch) + batch.map { |upload| verify(upload) }.compact.to_h end def verify(object) + local?(object) ? verify_local(object) : verify_remote(object) + rescue => err + failure(object, err.inspect) + end + + def verify_local(object) expected = expected_checksum(object) actual = actual_checksum(object) - raise 'Checksum missing' unless expected.present? - raise 'Checksum mismatch' unless expected == actual + return failure(object, 'Checksum missing') unless expected.present? + return failure(object, 'Checksum mismatch') unless expected == actual + + success + end + # We don't calculate checksum for remote objects, so just check existence + def verify_remote(object) + return failure(object, 'Remote object does not exist') unless remote_object_exists?(object) + + success + end + + def success nil - rescue => err - [object, err] + end + + def failure(object, message) + [object, message] + end + + # It's already set to Logger::INFO, but acts as if it is set to + # Logger::DEBUG, and this fixes it... + def fix_google_api_logger + if Object.const_defined?('Google::Apis') + Google::Apis.logger.level = Logger::INFO + end end # This should return an ActiveRecord::Relation suitable for calling #in_batches on - def relation + def all_relation + raise NotImplementedError.new + end + + # Should return true if the object is stored locally + def local?(_object) raise NotImplementedError.new end @@ -59,6 +93,11 @@ module Gitlab def actual_checksum(_object) raise NotImplementedError.new end + + # Be sure to perform a hard check of the remote object (don't just check DB value) + def remote_object_exists?(object) + raise NotImplementedError.new + end end end end diff --git a/lib/gitlab/verify/job_artifacts.rb b/lib/gitlab/verify/job_artifacts.rb index 03500a61074..dbadfbde9e3 100644 --- a/lib/gitlab/verify/job_artifacts.rb +++ b/lib/gitlab/verify/job_artifacts.rb @@ -11,10 +11,14 @@ module Gitlab private - def relation + def all_relation ::Ci::JobArtifact.all end + def local?(artifact) + artifact.local_store? + end + def expected_checksum(artifact) artifact.file_sha256 end @@ -22,6 +26,10 @@ module Gitlab def actual_checksum(artifact) Digest::SHA256.file(artifact.file.path).hexdigest end + + def remote_object_exists?(artifact) + artifact.file.file.exists? + end end end end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb index 970e2a7b718..d3f58a73ac7 100644 --- a/lib/gitlab/verify/lfs_objects.rb +++ b/lib/gitlab/verify/lfs_objects.rb @@ -11,8 +11,12 @@ module Gitlab private - def relation - LfsObject.with_files_stored_locally + def all_relation + LfsObject.all + end + + def local?(lfs_object) + lfs_object.local_store? end def expected_checksum(lfs_object) @@ -22,6 +26,10 @@ module Gitlab def actual_checksum(lfs_object) LfsObject.calculate_oid(lfs_object.file.path) end + + def remote_object_exists?(lfs_object) + lfs_object.file.file.exists? + end end end end diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb index dd138e6b92b..e190eaddc79 100644 --- a/lib/gitlab/verify/rake_task.rb +++ b/lib/gitlab/verify/rake_task.rb @@ -45,7 +45,7 @@ module Gitlab return unless verbose? failures.each do |object, error| - say " - #{verifier.describe(object)}: #{error.inspect}".color(:red) + say " - #{verifier.describe(object)}: #{error}".color(:red) end end end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 0ffa71a6d72..73fc43cb590 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -11,8 +11,12 @@ module Gitlab private - def relation - Upload.with_files_stored_locally + def all_relation + Upload.all.preload(:model) + end + + def local?(upload) + upload.local? end def expected_checksum(upload) @@ -22,6 +26,10 @@ module Gitlab def actual_checksum(upload) Upload.hexdigest(upload.absolute_path) end + + def remote_object_exists?(upload) + upload.build_uploader.file.exists? + end end end end |