diff options
Diffstat (limited to 'lib/gitlab')
26 files changed, 436 insertions, 145 deletions
diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 34286900e72..865185eb5db 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -6,7 +6,7 @@ module Gitlab module Auth module LDAP class Access - attr_reader :provider, :user + attr_reader :provider, :user, :ldap_identity def self.open(user, &block) Gitlab::Auth::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -14,9 +14,12 @@ module Gitlab end end - def self.allowed?(user) + def self.allowed?(user, options = {}) self.open(user) do |access| + # Whether user is allowed, or not, we should update + # permissions to keep things clean if access.allowed? + access.update_user Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute true @@ -29,7 +32,8 @@ module Gitlab def initialize(user, adapter = nil) @adapter = adapter @user = user - @provider = user.ldap_identity.provider + @ldap_identity = user.ldap_identity + @provider = adapter&.provider || ldap_identity&.provider end def allowed? @@ -40,7 +44,7 @@ module Gitlab end # Block user in GitLab if he/she was blocked in AD - if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) + if Gitlab::Auth::LDAP::Person.disabled_via_active_directory?(ldap_identity.extern_uid, adapter) block_user(user, 'is disabled in Active Directory') false else @@ -64,27 +68,44 @@ module Gitlab Gitlab::Auth::LDAP::Config.new(provider) end + def find_ldap_user + Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter) + end + def ldap_user - @ldap_user ||= Gitlab::Auth::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + return unless provider + + @ldap_user ||= find_ldap_user end def block_user(user, reason) user.ldap_block - Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" - ) + if provider + Gitlab::AppLogger.info( + "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + else + Gitlab::AppLogger.info( + "Account is not provided by LDAP, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end end def unblock_user(user, reason) user.activate Gitlab::AppLogger.info( - "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ "unblocking Gitlab user \"#{user.name}\" (#{user.email})" ) end + + def update_user + # no-op in CE + end end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 77185f52ced..d4415eaa6dc 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -11,6 +11,8 @@ module Gitlab attr_accessor :provider, :options + InvalidProvider = Class.new(StandardError) + def self.enabled? Gitlab.config.ldap.enabled end @@ -22,6 +24,10 @@ module Gitlab def self.available_servers return [] unless enabled? + _available_servers + end + + def self._available_servers Array.wrap(servers.first) end @@ -34,7 +40,7 @@ module Gitlab end def self.invalid_provider(provider) - raise "Unknown provider (#{provider}). Available providers: #{providers}" + raise InvalidProvider.new("Unknown provider (#{provider}). Available providers: #{providers}") end def initialize(provider) @@ -84,13 +90,17 @@ module Gitlab end def base - options['base'] + @base ||= Person.normalize_dn(options['base']) end def uid options['uid'] end + def label + options['label'] + end + def sync_ssh_keys? sync_ssh_keys.present? end @@ -132,6 +142,10 @@ module Gitlab options['timeout'].to_i end + def external_groups + options['external_groups'] || [] + end + def has_auth? options['password'] || options['bind_dn'] end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 2760b1a3247..5fa9581f837 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -14,6 +14,10 @@ module Gitlab def external_groups options[:external_groups] end + + def admin_groups + options[:admin_groups] + end end end end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index cb01cd8004c..b8c84c37cd5 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -20,10 +20,8 @@ module Gitlab user ||= find_or_build_ldap_user if auto_link_ldap_user? user ||= build_new_user if signup_enabled? - if external_users_enabled? && user - # Check if there is overlap between the user's groups and the external groups - # setting then set user as external or internal. - user.external = !(auth_hash.groups & saml_config.external_groups).empty? + if user + user.external = !(auth_hash.groups & saml_config.external_groups).empty? if external_users_enabled? end user diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index cf02030c577..4dc23f977da 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -1,9 +1,5 @@ module Gitlab module Auth - # - # Exceptions - # - AuthenticationError = Class.new(StandardError) MissingTokenError = Class.new(AuthenticationError) TokenNotFoundError = Class.new(AuthenticationError) @@ -61,6 +57,12 @@ module Gitlab private + def route_authentication_setting + return {} unless respond_to?(:route_setting) + + route_setting(:authentication) || {} + end + def access_token strong_memoize(:access_token) do find_oauth_access_token || find_personal_access_token diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb new file mode 100644 index 00000000000..f57df7c5637 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression.rb @@ -0,0 +1,10 @@ +module Gitlab + module Ci + module Pipeline + module Expression + ExpressionError = Class.new(StandardError) + RuntimeError = Class.new(ExpressionError) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb new file mode 100644 index 00000000000..10957598f76 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Matches < Lexeme::Operator + PATTERN = /=~/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + text = @left.evaluate(variables) + regexp = @right.evaluate(variables) + + regexp.scan(text.to_s).any? + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb new file mode 100644 index 00000000000..9b239c29ea4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + require_dependency 're2' + + class Pattern < Lexeme::Value + PATTERN = %r{^/.+/[ismU]*$}.freeze + + def initialize(regexp) + @value = regexp + + unless Gitlab::UntrustedRegexp.valid?(@value) + raise Lexer::SyntaxError, 'Invalid regular expression!' + end + end + + def evaluate(variables = {}) + Gitlab::UntrustedRegexp.fabricate(@value) + rescue RegexpError + raise Expression::RuntimeError, 'Invalid regular expression!' + end + + def self.build(string) + new(string) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index e1c68b7c3c2..4cacb1e62c9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -5,15 +5,17 @@ module Gitlab class Lexer include ::Gitlab::Utils::StrongMemoize + SyntaxError = Class.new(Expression::ExpressionError) + LEXEMES = [ Expression::Lexeme::Variable, Expression::Lexeme::String, + Expression::Lexeme::Pattern, Expression::Lexeme::Null, - Expression::Lexeme::Equals + Expression::Lexeme::Equals, + Expression::Lexeme::Matches ].freeze - SyntaxError = Class.new(Statement::StatementError) - MAX_TOKENS = 100 def initialize(statement, max_tokens: MAX_TOKENS) diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 09a7c98464b..b36f1e0f865 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -3,15 +3,16 @@ module Gitlab module Pipeline module Expression class Statement - StatementError = Class.new(StandardError) + StatementError = Class.new(Expression::ExpressionError) GRAMMAR = [ + %w[variable], %w[variable equals string], %w[variable equals variable], %w[variable equals null], %w[string equals variable], %w[null equals variable], - %w[variable] + %w[variable matches pattern] ].freeze def initialize(statement, variables = {}) @@ -35,11 +36,13 @@ module Gitlab def truthful? evaluate.present? + rescue Expression::ExpressionError + false end def valid? parse_tree.is_a?(Lexeme::Base) - rescue StatementError + rescue Expression::ExpressionError false end end diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb new file mode 100644 index 00000000000..e7a2e5511cf --- /dev/null +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + # Class for preloading data associated with pipelines such as commit + # authors. + module Preloader + def self.preload(pipelines) + # This ensures that all the pipeline commits are eager loaded before we + # start using them. + pipelines.each(&:commit) + + pipelines.each do |pipeline| + # This preloads the author of every commit. We're using "lazy_author" + # here since "author" immediately loads the data on the first call. + pipeline.commit.try(:lazy_author) + + # This preloads the number of warnings for every pipeline, ensuring + # that Ci::Pipeline#has_warnings? doesn't execute any additional + # queries. + pipeline.number_of_warnings + end + end + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 76501dd50e8..d49d055c3f2 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -43,7 +43,7 @@ module Gitlab end def self.version - database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] + @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end def self.join_lateral_supported? diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index d78d29b7ac6..156d077a69c 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -104,25 +104,22 @@ module Gitlab # file.rb # oid: 4a # # - # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a' + # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' # - def find_entry_by_path(repository, root_id, path) + def find_entry_by_path(repository, root_id, *path_parts) root_tree = repository.lookup(root_id) - # Strip leading slashes - path[%r{^/*}] = '' - path_arr = path.split('/') entry = root_tree.find do |entry| - entry[:name] == path_arr[0] + entry[:name] == path_parts[0] end return nil unless entry - if path_arr.size > 1 + if path_parts.size > 1 return nil unless entry[:type] == :tree - path_arr.shift - find_entry_by_path(repository, entry[:oid], path_arr.join('/')) + path_parts.shift + find_entry_by_path(repository, entry[:oid], *path_parts) else [:blob, :commit].include?(entry[:type]) ? entry : nil end @@ -185,10 +182,13 @@ module Gitlab def find_by_rugged(repository, sha, path, limit:) return unless path + # Strip any leading / characters from the path + path = path.sub(%r{\A/*}, '') + rugged_commit = repository.lookup(sha) root_tree = rugged_commit.tree - blob_entry = find_entry_by_path(repository, root_tree.oid, path) + blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) return nil unless blob_entry diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index fabcd46c8e9..89f761dd515 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -6,6 +6,7 @@ module Gitlab attr_accessor :raw_commit, :head + MAX_COMMIT_MESSAGE_DISPLAY_SIZE = 10.megabytes MIN_SHA_LENGTH = 7 SERIALIZE_KEYS = [ :id, :message, :parent_ids, @@ -63,9 +64,7 @@ module Gitlab if is_enabled repo.gitaly_commit_client.find_commit(commit_id) else - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(Rugged::Commit) ? obj : nil + rugged_find(repo, commit_id) end end @@ -76,6 +75,12 @@ module Gitlab nil end + def rugged_find(repo, commit_id) + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(Rugged::Commit) ? obj : nil + end + # Get last commit for HEAD # # Ex. @@ -297,11 +302,40 @@ module Gitlab nil end end + + def get_message(repository, commit_id) + BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + commit_ids = items.map { |i| i[:commit_id] } + + messages = get_messages(repository, commit_ids) + + messages.each do |commit_sha, message| + loader.call({ repository: repository, commit_id: commit_sha }, message) + end + end + end + end + + def get_messages(repository, commit_ids) + repository.gitaly_migrate(:commit_messages) do |is_enabled| + if is_enabled + repository.gitaly_commit_client.get_commit_messages(commit_ids) + else + commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h + end + end + end end def initialize(repository, raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit + @repository = repository + @head = head + case raw_commit when Hash init_from_hash(raw_commit) @@ -312,9 +346,6 @@ module Gitlab else raise "Invalid raw commit type: #{raw_commit.class}" end - - @repository = repository - @head = head end def sha @@ -342,21 +373,6 @@ module Gitlab parent_ids.first end - # Shows the diff between the commit's parent and the commit. - # - # Cuts out the header and stats from #to_patch and returns only the diff. - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 - def to_diff - Gitlab::GitalyClient.migrate(:commit_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.patch(id) - else - rugged_diff_from_parent.patch - end - end - end - # Returns a diff object for the changes from this commit's first parent. # If there is no parent, then the diff is between this commit and an # empty repo. See Repository#diff for keys allowed in the +options+ @@ -432,16 +448,6 @@ module Gitlab Gitlab::Git::CommitStats.new(@repository, self) end - def to_patch(options = {}) - begin - rugged_commit.to_mbox(options) - rescue Rugged::InvalidError => ex - if ex.message =~ /commit \w+ is a merge commit/i - 'Patch format is not currently supported for merge commits.' - end - end - end - # Get ref names collection # # Ex. @@ -543,7 +549,7 @@ module Gitlab # TODO: Once gitaly "takes over" Rugged consider separating the # subject from the message to make it clearer when there's one # available but not the other. - @message = (commit.body.presence || commit.subject).dup + @message = message_from_gitaly_body @authored_date = Time.at(commit.author.date.seconds).utc @author_name = commit.author.name.dup @author_email = commit.author.email.dup @@ -595,6 +601,25 @@ module Gitlab def refs(repo) repo.refs_hash[id] end + + def message_from_gitaly_body + return @raw_commit.subject.dup if @raw_commit.body_size.zero? + return @raw_commit.body.dup if full_body_fetched_from_gitaly? + + if @raw_commit.body_size > MAX_COMMIT_MESSAGE_DISPLAY_SIZE + "#{@raw_commit.subject}\n\n--commit message is too big".strip + else + fetch_body_from_gitaly + end + end + + def full_body_fetched_from_gitaly? + @raw_commit.body.bytesize == @raw_commit.body_size + end + + def fetch_body_from_gitaly + self.class.get_message(@repository, id) + end end end end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 155cf52f050..57b82a37d6c 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -6,7 +6,7 @@ module Gitlab class << self def normalize_path(filename) # Strip all leading slashes so that //foo -> foo - filename[%r{^/*}] = '' + filename = filename.sub(%r{\A/*}, '') # Expand relative paths (e.g. foo/../bar) filename = Pathname.new(filename) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 061865a7acf..2be604a5b13 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -403,10 +403,10 @@ module Gitlab prefix = archive_prefix(ref, commit.id, append_sha: append_sha) { - 'RepoPath' => path, 'ArchivePrefix' => prefix, 'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format), - 'CommitId' => commit.id + 'CommitId' => commit.id, + 'GitalyRepository' => gitaly_repository.to_h } end @@ -1473,10 +1473,19 @@ module Gitlab def search_files_by_content(query, ref) return [] if empty? || query.blank? - offset = 2 - args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) + safe_query = Regexp.escape(query) + ref ||= root_ref + + gitaly_migrate(:search_files_by_content) do |is_enabled| + if is_enabled + gitaly_repository_client.search_files_by_content(ref, safe_query) + else + offset = 2 + args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref}) - run_git(args).first.scrub.split(/^--\n/) + run_git(args).first.scrub.split(/^--\n/) + end + end end def can_be_merged?(source_sha, target_branch) @@ -1491,12 +1500,19 @@ module Gitlab def search_files_by_name(query, ref) safe_query = Regexp.escape(query.sub(%r{^/*}, "")) + ref ||= root_ref return [] if empty? || safe_query.blank? - args = %W(ls-tree -r --name-status --full-tree #{ref || root_ref} -- #{safe_query}) + gitaly_migrate(:search_files_by_name) do |is_enabled| + if is_enabled + gitaly_repository_client.search_files_by_name(ref, safe_query) + else + args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query}) - run_git(args).first.lines.map(&:strip) + run_git(args).first.lines.map(&:strip) + end + end end def find_commits_by_message(query, ref, path, limit, offset) @@ -1572,14 +1588,12 @@ module Gitlab end def checksum - gitaly_migrate(:calculate_checksum, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.calculate_checksum - else - calculate_checksum_by_shelling_out - end - end + # The exists? RPC is much cheaper, so we perform this request first + raise NoRepository, "Repository does not exists" unless exists? + + gitaly_repository_client.calculate_checksum + rescue GRPC::NotFound + raise NoRepository # Guard against data races. end private @@ -1948,7 +1962,12 @@ module Gitlab end target_commit = Gitlab::Git::Commit.find(self, ref.target) - Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) + Gitlab::Git::Tag.new(self, { + name: ref.name, + target: ref.target, + target_commit: target_commit, + message: message + }) end.sort_by(&:name) end @@ -2348,7 +2367,7 @@ module Gitlab end def gitaly_delete_refs(*ref_names) - gitaly_ref_client.delete_refs(refs: ref_names) + gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end def rugged_remove_remote(remote_name) @@ -2478,36 +2497,6 @@ module Gitlab rev_parse_target(ref).oid end - def calculate_checksum_by_shelling_out - raise NoRepository unless exists? - - args = %W(--git-dir=#{path} show-ref --heads --tags) - output, status = run_git(args) - - if status.nil? || !status.zero? - # Non-valid git repositories return 128 as the status code and an error output - raise InvalidRepository if status == 128 && output.to_s.downcase =~ /not a git repository/ - # Empty repositories returns with a non-zero status and an empty output. - raise ChecksumError, output unless output.blank? - - return EMPTY_REPOSITORY_CHECKSUM - end - - refs = output.split("\n") - - result = refs.inject(nil) do |checksum, ref| - value = Digest::SHA1.hexdigest(ref).hex - - if checksum.nil? - value - else - checksum ^ value - end - end - - result.to_s(16) - end - def build_git_cmd(*args) object_directories = alternate_object_directories.join(File::PATH_SEPARATOR) diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 8a01f92e2af..e35ea5762eb 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -35,7 +35,11 @@ module Gitlab next if name =~ /\^\{\}\Z/ target_commit = Gitlab::Git::Commit.find(self, target) - Gitlab::Git::Tag.new(self, name, target, target_commit) + Gitlab::Git::Tag.new(self, { + name: name, + target: target, + target_commit: target_commit + }) end.compact end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 8a8f7b051ed..e44284572fd 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,17 +1,99 @@ module Gitlab module Git class Tag < Ref - attr_reader :object_sha + extend Gitlab::EncodingHelper + + attr_reader :object_sha, :repository + + MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes + SERIALIZE_KEYS = %i[name target target_commit message].freeze + + attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + + class << self + def get_message(repository, tag_id) + BatchLoader.for({ repository: repository, tag_id: tag_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + tag_ids = items.map { |i| i[:tag_id] } + + messages = get_messages(repository, tag_ids) + + messages.each do |id, message| + loader.call({ repository: repository, tag_id: id }, message) + end + end + end + end + + def get_messages(repository, tag_ids) + repository.gitaly_migrate(:tag_messages) do |is_enabled| + if is_enabled + repository.gitaly_ref_client.get_tag_messages(tag_ids) + else + tag_ids.map do |id| + tag = repository.rugged.lookup(id) + message = tag.is_a?(Rugged::Commit) ? "" : tag.message + + [id, message] + end.to_h + end + end + end + end + + def initialize(repository, raw_tag) + @repository = repository + @raw_tag = raw_tag + + case raw_tag + when Hash + init_from_hash + when Gitaly::Tag + init_from_gitaly + end - def initialize(repository, name, target, target_commit, message = nil) super(repository, name, target, target_commit) + end + + def init_from_hash + raw_tag = @raw_tag.symbolize_keys + + SERIALIZE_KEYS.each do |key| + send("#{key}=", raw_tag[key]) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def init_from_gitaly + @name = encode!(@raw_tag.name.dup) + @target = @raw_tag.id + @message = message_from_gitaly_tag - @message = message + if @raw_tag.target_commit.present? + @target_commit = Gitlab::Git::Commit.decorate(repository, @raw_tag.target_commit) + end end def message encode! @message end + + private + + def message_from_gitaly_tag + return @raw_tag.message.dup if full_message_fetched_from_gitaly? + + if @raw_tag.message_size > MAX_TAG_MESSAGE_DISPLAY_SIZE + '--tag message is too big' + else + self.class.get_message(@repository, target) + end + end + + def full_message_fetched_from_gitaly? + @raw_tag.message.bytesize == @raw_tag.message_size + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index a36e6c822f9..1f5f88bf792 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -334,6 +334,22 @@ module Gitlab signatures end + def get_commit_messages(commit_ids) + request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request) + + messages = Hash.new { |h, k| h[k] = ''.b } + current_commit_id = nil + + response.each do |rpc_message| + current_commit_id = rpc_message.commit_id if rpc_message.commit_id.present? + + messages[current_commit_id] << rpc_message.message + end + + messages + end + private def call_commit_diff(request_params, options = {}) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 831cfd1e014..44b0e517bf0 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -40,7 +40,7 @@ module Gitlab raise Gitlab::Git::Repository::TagExistsError end - Util.gitlab_tag_from_gitaly_tag(@repository, response.tag) + Gitlab::Git::Tag.new(@repository, response.tag) rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Repository::InvalidRef, e end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index ba6b577fd17..3ac46be6208 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -171,6 +171,22 @@ module Gitlab consume_ref_contains_sha_response(stream, :branch_names) end + def get_tag_messages(tag_ids) + request = Gitaly::GetTagMessagesRequest.new(repository: @gitaly_repo, tag_ids: tag_ids) + response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request) + + messages = Hash.new { |h, k| h[k] = ''.b } + current_tag_id = nil + + response.each do |rpc_message| + current_tag_id = rpc_message.tag_id if rpc_message.tag_id.present? + + messages[current_tag_id] << rpc_message.message + end + + messages + end + private def consume_refs_response(response) @@ -210,7 +226,7 @@ module Gitlab def consume_tags_response(response) response.flat_map do |message| - message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } + message.tags.map { |gitaly_tag| Gitlab::Git::Tag.new(@repository, gitaly_tag) } end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 132a5947f17..ee01f5a5bd9 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -301,6 +301,16 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :get_raw_changes, request) end + + def search_files_by_name(ref, query) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query) + GitalyClient.call(@storage, :repository_service, :search_files_by_name, request).flat_map(&:files) + end + + def search_files_by_content(ref, query) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) + GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches) + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 405567db94a..9c19c51d412 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -21,20 +21,6 @@ module Gitlab gitaly_repository.relative_path, gitaly_repository.gl_repository) end - - def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) - if gitaly_tag.target_commit.present? - commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) - end - - Gitlab::Git::Tag.new( - repository, - Gitlab::EncodingHelper.encode!(gitaly_tag.name.dup), - gitaly_tag.id, - commit, - Gitlab::EncodingHelper.encode!(gitaly_tag.message.chomp) - ) - end end end end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index c9122a23568..d323cb9dadf 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -57,7 +57,7 @@ module Gitlab regex = Regexp.escape(wildcard_address) regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') - Regexp.new(regex).freeze + Regexp.new(/\A#{regex}\z/).freeze end end end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 75ba0799058..dc2d91dfa23 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -9,7 +9,9 @@ module Gitlab # there is a strict limit on total execution time. See the RE2 documentation # at https://github.com/google/re2/wiki/Syntax for more details. class UntrustedRegexp - delegate :===, to: :regexp + require_dependency 're2' + + delegate :===, :source, to: :regexp def initialize(pattern, multiline: false) if multiline @@ -35,6 +37,10 @@ module Gitlab RE2.Replace(text, regexp, rewrite) end + def ==(other) + self.source == other.source + end + # Handles regular expressions with the preferred RE2 library where possible # via UntustedRegex. Falls back to Ruby's built-in regular expression library # when the syntax would be invalid in RE2. @@ -48,6 +54,24 @@ module Gitlab Regexp.new(pattern) end + def self.valid?(pattern) + !!self.fabricate(pattern) + rescue RegexpError + false + end + + def self.fabricate(pattern) + matches = pattern.match(%r{^/(?<regexp>.+)/(?<flags>[ismU]*)$}) + + raise RegexpError, 'Invalid regular expression!' if matches.nil? + + expression = matches[:regexp] + flags = matches[:flags] + expression.prepend("(?#{flags})") if flags.present? + + self.new(expression, multiline: false) + end + private attr_reader :regexp diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 1f060de657d..e893e46ee86 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -65,12 +65,7 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) raise "Repository or ref not found" if params.empty? - if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - params.merge!( - 'GitalyServer' => gitaly_server_hash(repository), - 'GitalyRepository' => repository.gitaly_repository.to_h - ) - end + params['GitalyServer'] = gitaly_server_hash(repository) # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. params['DisableCache'] = true if git_archive_cache_disabled? |