diff options
Diffstat (limited to 'lib/gitlab')
45 files changed, 1657 insertions, 93 deletions
diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb new file mode 100644 index 00000000000..c88eb9783ed --- /dev/null +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -0,0 +1,65 @@ +module Gitlab + module BackgroundMigration + class CreateForkNetworkMembershipsRange + RESCHEDULE_DELAY = 15 + + class ForkedProjectLink < ActiveRecord::Base + self.table_name = 'forked_project_links' + end + + def perform(start_id, end_id) + log("Creating memberships for forks: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS + INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) + + SELECT fork_network_members.fork_network_id, + forked_project_links.forked_to_project_id, + forked_project_links.forked_from_project_id + + FROM forked_project_links + + INNER JOIN fork_network_members + ON forked_project_links.forked_from_project_id = fork_network_members.project_id + + WHERE forked_project_links.id BETWEEN #{start_id} AND #{end_id} + AND NOT EXISTS ( + SELECT true + FROM fork_network_members existing_members + WHERE existing_members.project_id = forked_project_links.forked_to_project_id + ) + INSERT_MEMBERS + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def missing_members?(start_id, end_id) + count_sql = <<~MISSING_MEMBERS + SELECT COUNT(*) + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = forked_project_links.forked_to_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE forked_project_links.forked_from_project_id = projects.id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + MISSING_MEMBERS + + ForkNetworkMember.count_by_sql(count_sql) > 0 + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb new file mode 100644 index 00000000000..e94719db72e --- /dev/null +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -0,0 +1,53 @@ +class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys + class GpgKey < ActiveRecord::Base + self.table_name = 'gpg_keys' + + include EachBatch + include ShaAttribute + + sha_attribute :primary_keyid + sha_attribute :fingerprint + + has_many :subkeys, class_name: 'GpgKeySubkey' + end + + class GpgKeySubkey < ActiveRecord::Base + self.table_name = 'gpg_key_subkeys' + + include ShaAttribute + + sha_attribute :keyid + sha_attribute :fingerprint + end + + def perform(gpg_key_id) + gpg_key = GpgKey.find_by(id: gpg_key_id) + + return if gpg_key.nil? + return if gpg_key.subkeys.any? + + create_subkeys(gpg_key) + update_signatures(gpg_key) + end + + private + + def create_subkeys(gpg_key) + gpg_subkeys = Gitlab::Gpg.subkeys_from_key(gpg_key.key) + + gpg_subkeys[gpg_key.primary_keyid.upcase]&.each do |subkey_data| + gpg_key.subkeys.build(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint]) + end + + # Improve latency by doing all INSERTs in a single call + GpgKey.transaction do + gpg_key.save! + end + end + + def update_signatures(gpg_key) + return unless gpg_key.subkeys.exists? + + InvalidGpgSignatureUpdateWorker.perform_async(gpg_key.id) + end +end diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb new file mode 100644 index 00000000000..bc53e6d7f94 --- /dev/null +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -0,0 +1,313 @@ +module Gitlab + module BackgroundMigration + class NormalizeLdapExternUidsRange + class Identity < ActiveRecord::Base + self.table_name = 'identities' + end + + # Copied this class to make this migration resilient to future code changes. + # And if the normalize behavior is changed in the future, it must be + # accompanied by another migration. + module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end + end + + def perform(start_id, end_id) + return unless migrate? + + ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) + ldap_identities.each do |identity| + begin + identity.extern_uid = Gitlab::LDAP::DN.new(identity.extern_uid).to_normalized_s + unless identity.save + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." + end + rescue Gitlab::LDAP::DN::FormatError => e + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." + end + end + end + + def migrate? + Identity.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb new file mode 100644 index 00000000000..2ef3a207dd3 --- /dev/null +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -0,0 +1,59 @@ +module Gitlab + module BackgroundMigration + class PopulateForkNetworksRange + def perform(start_id, end_id) + log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_from_project_id + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + + log("Creating memberships for root projects: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_ROOT + INSERT INTO fork_network_members (fork_network_id, project_id) + SELECT DISTINCT fork_networks.id, fork_networks.root_project_id + + FROM fork_networks + + INNER JOIN forked_project_links + ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = fork_networks.root_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_ROOT + + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY + BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 088adbdd267..72b75791bbb 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -155,7 +155,7 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? - if s.scan(/section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/) + if s.scan(Gitlab::Regex.build_trace_section_regex) handle_section(s) elsif s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb index 489bcd79655..075504bcce5 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -13,7 +13,7 @@ module Gitlab end if @command.save_incompleted && @pipeline.has_yaml_errors? - @pipeline.drop + @pipeline.drop!(:config_error) end return error(@pipeline.yaml_errors) diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index e19aae35a81..bc97aa63b02 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -3,7 +3,9 @@ module Gitlab module Stage class Seed attr_reader :pipeline + delegate :project, to: :pipeline + delegate :size, to: :@jobs def initialize(pipeline, stage, jobs) @pipeline = pipeline diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 5b835bb669a..baf55b1fa07 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -27,6 +27,12 @@ module Gitlab end end + def extract_sections + read do |stream| + stream.extract_sections + end + end + def set(data) write do |stream| data = job.hide_secrets(data) diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb new file mode 100644 index 00000000000..9bb0166c9e3 --- /dev/null +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -0,0 +1,97 @@ +module Gitlab + module Ci + class Trace + class SectionParser + def initialize(lines) + @lines = lines + end + + def parse! + @markers = {} + + @lines.each do |line, pos| + parse_line(line, pos) + end + end + + def sections + sanitize_markers.map do |name, markers| + start_, end_ = markers + + { + name: name, + byte_start: start_[:marker], + byte_end: end_[:marker], + date_start: start_[:timestamp], + date_end: end_[:timestamp] + } + end + end + + private + + def parse_line(line, line_start_position) + s = StringScanner.new(line) + until s.eos? + find_next_marker(s) do |scanner| + marker_begins_at = line_start_position + scanner.pointer + + if scanner.scan(Gitlab::Regex.build_trace_section_regex) + marker_ends_at = line_start_position + scanner.pointer + handle_line(scanner[1], scanner[2].to_i, scanner[3], marker_begins_at, marker_ends_at) + true + else + false + end + end + end + end + + def sanitize_markers + @markers.select do |_, markers| + markers.size == 2 && markers[0][:action] == :start && markers[1][:action] == :end + end + end + + def handle_line(action, time, name, marker_start, marker_end) + action = action.to_sym + timestamp = Time.at(time).utc + marker = if action == :start + marker_end + else + marker_start + end + + @markers[name] ||= [] + @markers[name] << { + name: name, + action: action, + timestamp: timestamp, + marker: marker + } + end + + def beginning_of_section_regex + @beginning_of_section_regex ||= /section_/.freeze + end + + def find_next_marker(s) + beginning_of_section_len = 8 + maybe_marker = s.exist?(beginning_of_section_regex) + + if maybe_marker.nil? + s.terminate + else + # repositioning at the beginning of the match + s.pos += maybe_marker - beginning_of_section_len + if block_given? + good_marker = yield(s) + # if not a good marker: Consuming the matched beginning_of_section_regex + s.pos += beginning_of_section_len unless good_marker + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index ab3408f48d6..d52194f688b 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -90,8 +90,25 @@ module Gitlab # so we just silently ignore error for now end + def extract_sections + return [] unless valid? + + lines = to_enum(:each_line_with_pos) + parser = SectionParser.new(lines) + + parser.parse! + parser.sections + end + private + def each_line_with_pos + stream.seek(0, IO::SEEK_SET) + stream.each_line do |line| + yield [line, stream.pos - line.bytesize] + end + end + def read_last_lines(limit) to_enum(:reverse_line).first(limit).reverse.join end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 243c1f1394d..7e7aaeeaa17 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -23,7 +23,8 @@ module Gitlab @extractor.analyze(closing_statements.join(" ")) @extractor.issues.reject do |issue| - @extractor.project.forked_from?(issue.project) # Don't extract issues on original project + # Don't extract issues from the project this project was forked from + @extractor.project.forked_from?(issue.project) end end end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 31a46a738c3..c169c8fe135 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -86,7 +86,7 @@ module Gitlab user_name: user.name, user_username: user.username, user_email: user.email, - user_avatar: user.avatar_url, + user_avatar: user.avatar_url(only_path: false), project_id: project.id, project: project.hook_attrs, commits: commit_attrs, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index a6ec75da385..357f16936c6 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -29,6 +29,15 @@ module Gitlab adapter_name.casecmp('postgresql').zero? end + # Overridden in EE + def self.read_only? + false + end + + def self.read_write? + !self.read_only? + end + def self.version database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index fcac85ff892..599c3c5deab 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -27,16 +27,23 @@ module Gitlab @fallback_diff_refs = fallback_diff_refs end - def position(line) + def position(position_marker, position_type: :text) return unless diff_refs - Position.new( + data = { + diff_refs: diff_refs, + position_type: position_type.to_s, old_path: old_path, - new_path: new_path, - old_line: line.old_line, - new_line: line.new_line, - diff_refs: diff_refs - ) + new_path: new_path + } + + if position_type == :text + data.merge!(text_position_properties(position_marker)) + else + data.merge!(image_position_properties(position_marker)) + end + + Position.new(data) end def line_code(line) @@ -228,6 +235,14 @@ module Gitlab private + def text_position_properties(line) + { old_line: line.old_line, new_line: line.new_line } + end + + def image_position_properties(image_point) + image_point.to_h + end + def blobs_changed? old_blob && new_blob && old_blob.id != new_blob.id end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb new file mode 100644 index 00000000000..5e923b9e602 --- /dev/null +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -0,0 +1,61 @@ +module Gitlab + module Diff + module Formatters + class BaseFormatter + attr_reader :old_path + attr_reader :new_path + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + attr_reader :position_type + + def initialize(attrs) + if diff_file = attrs[:diff_file] + attrs[:diff_refs] = diff_file.diff_refs + attrs[:old_path] = diff_file.old_path + attrs[:new_path] = diff_file.new_path + end + + if diff_refs = attrs[:diff_refs] + attrs[:base_sha] = diff_refs.base_sha + attrs[:start_sha] = diff_refs.start_sha + attrs[:head_sha] = diff_refs.head_sha + end + + @old_path = attrs[:old_path] + @new_path = attrs[:new_path] + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + end + + def key + [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || "")] + end + + def to_h + { + base_sha: base_sha, + start_sha: start_sha, + head_sha: head_sha, + old_path: old_path, + new_path: new_path, + position_type: position_type + } + end + + def position_type + raise NotImplementedError + end + + def ==(other) + raise NotImplementedError + end + + def complete? + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb new file mode 100644 index 00000000000..ccd0d309972 --- /dev/null +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -0,0 +1,43 @@ +module Gitlab + module Diff + module Formatters + class ImageFormatter < BaseFormatter + attr_reader :width + attr_reader :height + attr_reader :x + attr_reader :y + + def initialize(attrs) + @x = attrs[:x] + @y = attrs[:y] + @width = attrs[:width] + @height = attrs[:height] + + super(attrs) + end + + def key + @key ||= super.push(x, y) + end + + def complete? + x && y && width && height + end + + def to_h + super.merge(width: width, height: height, x: x, y: y) + end + + def position_type + "image" + end + + def ==(other) + other.is_a?(self.class) && + x == other.x && + y == other.y + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb new file mode 100644 index 00000000000..01c7e9f51ab --- /dev/null +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -0,0 +1,49 @@ +module Gitlab + module Diff + module Formatters + class TextFormatter < BaseFormatter + attr_reader :old_line + attr_reader :new_line + + def initialize(attrs) + @old_line = attrs[:old_line] + @new_line = attrs[:new_line] + + super(attrs) + end + + def key + @key ||= super.push(old_line, new_line) + end + + def complete? + old_line || new_line + end + + def to_h + super.merge(old_line: old_line, new_line: new_line) + end + + def line_age + if old_line && new_line + nil + elsif new_line + 'new' + else + 'old' + end + end + + def position_type + "text" + end + + def ==(other) + other.is_a?(self.class) && + new_line == other.new_line && + old_line == other.old_line + end + end + end + end +end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb new file mode 100644 index 00000000000..65332dfd239 --- /dev/null +++ b/lib/gitlab/diff/image_point.rb @@ -0,0 +1,23 @@ +module Gitlab + module Diff + class ImagePoint + attr_reader :width, :height, :x, :y + + def initialize(width, height, x, y) + @width = width + @height = height + @x = x + @y = y + end + + def to_h + { + width: width, + height: height, + x: x, + y: y + } + end + end + end +end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index b8db3adef0a..bd0a9502a5e 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -1,37 +1,25 @@ -# Defines a specific location, identified by paths and line numbers, +# Defines a specific location, identified by paths line numbers and image coordinates, # within a specific diff, identified by start, head and base commit ids. module Gitlab module Diff class Position - attr_reader :old_path - attr_reader :new_path - attr_reader :old_line - attr_reader :new_line - attr_reader :base_sha - attr_reader :start_sha - attr_reader :head_sha - + attr_accessor :formatter + + delegate :old_path, + :new_path, + :base_sha, + :start_sha, + :head_sha, + :old_line, + :new_line, + :position_type, to: :formatter + + # A position can belong to a text line or to an image coordinate + # it depends of the position_type argument. + # Text position will have: new_line and old_line + # Image position will have: width, height, x, y def initialize(attrs = {}) - if diff_file = attrs[:diff_file] - attrs[:diff_refs] = diff_file.diff_refs - attrs[:old_path] = diff_file.old_path - attrs[:new_path] = diff_file.new_path - end - - if diff_refs = attrs[:diff_refs] - attrs[:base_sha] = diff_refs.base_sha - attrs[:start_sha] = diff_refs.start_sha - attrs[:head_sha] = diff_refs.head_sha - end - - @old_path = attrs[:old_path] - @new_path = attrs[:new_path] - @base_sha = attrs[:base_sha] - @start_sha = attrs[:start_sha] - @head_sha = attrs[:head_sha] - - @old_line = attrs[:old_line] - @new_line = attrs[:new_line] + @formatter = get_formatter_class(attrs[:position_type]).new(attrs) end # `Gitlab::Diff::Position` objects are stored as serialized attributes in @@ -46,7 +34,11 @@ module Gitlab end def encode_with(coder) - coder['attributes'] = self.to_h + coder['attributes'] = formatter.to_h + end + + def key + formatter.key end def ==(other) @@ -54,20 +46,11 @@ module Gitlab other.diff_refs == diff_refs && other.old_path == old_path && other.new_path == new_path && - other.old_line == old_line && - other.new_line == new_line + other.formatter == formatter end def to_h - { - old_path: old_path, - new_path: new_path, - old_line: old_line, - new_line: new_line, - base_sha: base_sha, - start_sha: start_sha, - head_sha: head_sha - } + formatter.to_h end def inspect @@ -75,23 +58,15 @@ module Gitlab end def complete? - file_path.present? && - (old_line || new_line) && - diff_refs.complete? + file_path.present? && formatter.complete? && diff_refs.complete? end def to_json(opts = nil) - JSON.generate(self.to_h, opts) + JSON.generate(formatter.to_h, opts) end def type - if old_line && new_line - nil - elsif new_line - 'new' - else - 'old' - end + formatter.line_age end def unchanged? @@ -150,6 +125,17 @@ module Gitlab diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first end + + def get_formatter_class(type) + type ||= "text" + + case type + when 'image' + Gitlab::Diff::Formatters::ImageFormatter + else + Gitlab::Diff::Formatters::TextFormatter + end + end end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index c5a8ea12245..c4c60d1dfee 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,7 +2,7 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze + DEFAULT_CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze @@ -20,7 +20,7 @@ module Gitlab attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found attr_reader :failed_files - def initialize(branch:, ce_repo: CE_REPO) + def initialize(branch:, ce_repo: DEFAULT_CE_REPO) @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch @@ -132,7 +132,7 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) step("Resetting to latest master", %w[git reset --hard origin/master]) - step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) + step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even diff --git a/lib/gitlab/gcp/model.rb b/lib/gitlab/gcp/model.rb new file mode 100644 index 00000000000..195391f0e3c --- /dev/null +++ b/lib/gitlab/gcp/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Gcp + module Model + def table_name_prefix + "gcp_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 89b654253cb..0f059bef808 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -990,7 +990,7 @@ module Gitlab tmp_ref = fetch_ref( start_repository, source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", - target_ref: "refs/tmp/#{SecureRandom.hex}/head" + target_ref: "refs/tmp/#{SecureRandom.hex}" ) yield commit(sha) @@ -1112,6 +1112,8 @@ module Gitlab raise NoRepository.new(e) rescue GRPC::BadStatus => e raise CommandError.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) end private diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index db67ede9d9e..42b59c106e2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -17,7 +17,8 @@ module Gitlab command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', - readonly: 'The repository is temporarily read-only. Please try again later.' + read_only: 'The repository is temporarily read-only. Please try again later.', + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -161,7 +162,11 @@ module Gitlab def check_push_access!(changes) if project.repository_read_only? - raise UnauthorizedError, ERROR_MESSAGES[:readonly] + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only] end if deploy_key diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1fe5155c093..98f1f45b338 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,7 @@ module Gitlab class GitAccessWiki < GitAccess ERROR_MESSAGES = { + read_only: "You can't push code to a read-only GitLab instance.", write_to_wiki: "You are not allowed to write to this project's wiki." }.freeze @@ -17,6 +18,10 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + true end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index a3c6b21a6a1..2e3e4fc3f1f 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -11,7 +11,7 @@ module Gitlab return false if ref_name.start_with?('refs/remotes/') Gitlab::Utils.system_silent( - %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format --branch #{ref_name})) end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 87b300dcf7e..cf36106e23d 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -28,6 +28,7 @@ module Gitlab SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze MAXIMUM_GITALY_CALLS = 30 + CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new private_constant :MUTEX @@ -79,7 +80,16 @@ module Gitlab def self.request_metadata(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) - { metadata: { 'authorization' => "Bearer #{encoded_token}" } } + metadata = { + 'authorization' => "Bearer #{encoded_token}", + 'client_name' => CLIENT_NAME + } + + feature_stack = Thread.current[:gitaly_feature_stack] + feature = feature_stack && feature_stack[0] + metadata['call_site'] = feature.to_s if feature + + { metadata: metadata } end def self.token(storage) @@ -137,7 +147,14 @@ module Gitlab Gitlab::Metrics.measure(metric_name) do # Some migrate calls wrap other migrate calls allow_n_plus_1_calls do - yield is_enabled + feature_stack = Thread.current[:gitaly_feature_stack] ||= [] + feature_stack.unshift(feature) + begin + yield is_enabled + ensure + feature_stack.shift + Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? + end end end end diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb new file mode 100644 index 00000000000..bd7c345ac01 --- /dev/null +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -0,0 +1,39 @@ +module Gitlab + module GitalyClient + class NamespaceService + def initialize(storage) + @storage = storage + end + + def exists?(name) + request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:namespace_exists, request).exists + end + + def add(name) + request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:add_namespace, request) + end + + def remove(name) + request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:remove_namespace, request) + end + + def rename(from, to) + request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to) + + gitaly_client_call(:rename_namespace, request) + end + + private + + def gitaly_client_call(type, request) + GitalyClient.call(@storage, :namespace_service, type, request) + end + end + end +end diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 0d5039ddf5f..413872d7e08 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -34,6 +34,21 @@ module Gitlab end end + def subkeys_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + raw_keys = GPGME::Key.find(:public, fingerprints) + + raw_keys.each_with_object({}) do |raw_key, grouped_subkeys| + primary_subkey_id = raw_key.primary_subkey.keyid + + grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..-1].map do |s| + { keyid: s.keyid, fingerprint: s.fingerprint } + end + end + end + end + def user_infos_from_key(key) using_tmp_keychain do fingerprints = CurrentKeyChain.fingerprints_from_key(key) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 86bd9f5b125..0f4ba6f83fc 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -43,7 +43,9 @@ module Gitlab # key belonging to the keyid. # This way we can add the key to the temporary keychain and extract # the proper signature. - gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + # NOTE: the invoked method is #fingerprint but it's only returning + # 16 characters (the format used by keyid) instead of 40. + gpg_key = find_gpg_key(verified_signature.fingerprint) if gpg_key Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) @@ -74,7 +76,7 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, gpg_key: gpg_key, - gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], verification_status: verification_status @@ -98,6 +100,10 @@ module Gitlab def user_infos(gpg_key) gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end + + def find_gpg_key(keyid) + GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) + end end end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index e085eab26c9..1991911ef6a 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -9,8 +9,8 @@ module Gitlab GpgSignature .select(:id, :commit_sha, :project_id) .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) - .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) - .find_each { |sig| sig.gpg_commit.update_signature!(sig) } + .where(gpg_key_primary_keyid: @gpg_key.keyids) + .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end end end diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb new file mode 100644 index 00000000000..4febb0ab430 --- /dev/null +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -0,0 +1,56 @@ +module Gitlab + module HookData + class IssuableBuilder + CHANGES_KEYS = %i[previous current].freeze + + attr_accessor :issuable + + def initialize(issuable) + @issuable = issuable + end + + def build(user: nil, changes: {}) + hook_data = { + object_kind: issuable.class.name.underscore, + user: user.hook_attrs, + project: issuable.project.hook_attrs, + object_attributes: issuable.hook_attrs, + labels: issuable.labels.map(&:hook_attrs), + changes: final_changes(changes.slice(*safe_keys)), + # DEPRECATED + repository: issuable.project.hook_attrs.slice(:name, :url, :description, :homepage) + } + + if issuable.is_a?(Issue) + hook_data[:assignees] = issuable.assignees.map(&:hook_attrs) if issuable.assignees.any? + else + hook_data[:assignee] = issuable.assignee.hook_attrs if issuable.assignee + end + + hook_data + end + + def safe_keys + issuable_builder::SAFE_HOOK_ATTRIBUTES + issuable_builder::SAFE_HOOK_RELATIONS + end + + private + + def issuable_builder + case issuable + when Issue + Gitlab::HookData::IssueBuilder + when MergeRequest + Gitlab::HookData::MergeRequestBuilder + end + end + + def final_changes(changes_hash) + changes_hash.reduce({}) do |hash, (key, changes_array)| + hash[key] = Hash[CHANGES_KEYS.zip(changes_array)] + hash + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb new file mode 100644 index 00000000000..de9cab80a02 --- /dev/null +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -0,0 +1,55 @@ +module Gitlab + module HookData + class IssueBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + assignee_id + author_id + branch_name + closed_at + confidential + created_at + deleted_at + description + due_date + id + iid + last_edited_at + last_edited_by_id + milestone_id + moved_to_id + project_id + relative_position + state + time_estimate + title + updated_at + updated_by_id + ].freeze + + SAFE_HOOK_RELATIONS = %i[ + assignees + labels + ].freeze + + attr_accessor :issue + + def initialize(issue) + @issue = issue + end + + def build + attrs = { + url: Gitlab::UrlBuilder.build(issue), + total_time_spent: issue.total_time_spent, + human_total_time_spent: issue.human_total_time_spent, + human_time_estimate: issue.human_time_estimate, + assignee_ids: issue.assignee_ids, + assignee_id: issue.assignee_ids.first # This key is deprecated + } + + issue.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) + .merge!(attrs) + end + end + end +end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb new file mode 100644 index 00000000000..eaef19c9d04 --- /dev/null +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -0,0 +1,62 @@ +module Gitlab + module HookData + class MergeRequestBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + assignee_id + author_id + created_at + deleted_at + description + head_pipeline_id + id + iid + last_edited_at + last_edited_by_id + merge_commit_sha + merge_error + merge_params + merge_status + merge_user_id + merge_when_pipeline_succeeds + milestone_id + ref_fetched + source_branch + source_project_id + state + target_branch + target_project_id + time_estimate + title + updated_at + updated_by_id + ].freeze + + SAFE_HOOK_RELATIONS = %i[ + assignee + labels + ].freeze + + attr_accessor :merge_request + + def initialize(merge_request) + @merge_request = merge_request + end + + def build + attrs = { + url: Gitlab::UrlBuilder.build(merge_request), + source: merge_request.source_project.try(:hook_attrs), + target: merge_request.target_project.hook_attrs, + last_commit: merge_request.diff_head_commit&.hook_attrs, + work_in_progress: merge_request.work_in_progress?, + total_time_spent: merge_request.total_time_spent, + human_total_time_spent: merge_request.human_total_time_spent, + human_time_estimate: merge_request.human_time_estimate + } + + merge_request.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) + .merge!(attrs) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2171c6c7bbb..dec8b4c5acd 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -53,6 +53,7 @@ project_tree: - :auto_devops - :triggers - :pipeline_schedules + - :cluster - :services - :hooks - protected_branches: diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 380b336395d..a76cf1addc0 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -8,6 +8,8 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + cluster: 'Gcp::Cluster', + clusters: 'Gcp::Cluster', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 4fbc5fa5262..3123da17fd9 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -3,6 +3,10 @@ module Gitlab module LDAP class AuthHash < Gitlab::OAuth::AuthHash + def uid + Gitlab::LDAP::Person.normalize_dn(super) + end + private def get_info(key) diff --git a/lib/gitlab/ldap/dn.rb b/lib/gitlab/ldap/dn.rb new file mode 100644 index 00000000000..d6142dc6549 --- /dev/null +++ b/lib/gitlab/ldap/dn.rb @@ -0,0 +1,301 @@ +# -*- ruby encoding: utf-8 -*- + +# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` +# +# For our purposes, this class is used to normalize DNs in order to allow proper +# comparison. +# +# E.g. DNs should be compared case-insensitively (in basically all LDAP +# implementations or setups), therefore we downcase every DN. + +## +# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN +# ("Distinguished Name") is a unique identifier for an entry within an LDAP +# directory. It is made up of a number of other attributes strung together, +# to identify the entry in the tree. +# +# Each attribute that makes up a DN needs to have its value escaped so that +# the DN is valid. This class helps take care of that. +# +# A fully escaped DN needs to be unescaped when analysing its contents. This +# class also helps take care of that. +module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end +end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 9a6f7827b16..38d7a9ba2f5 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -36,6 +36,26 @@ module Gitlab ] end + def self.normalize_dn(dn) + ::Gitlab::LDAP::DN.new(dn).to_normalized_s + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + + dn + end + + # Returns the UID in a normalized form. + # + # 1. Excess spaces are stripped + # 2. The string is downcased (for case-insensitivity) + def self.normalize_uid(uid) + ::Gitlab::LDAP::DN.normalize_value(uid) + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + + uid + end + def initialize(entry, provider) Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @@ -58,7 +78,9 @@ module Gitlab attribute_value(:email) end - delegate :dn, to: :entry + def dn + self.class.normalize_dn(entry.dn) + end private diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb new file mode 100644 index 00000000000..0de0cddcce4 --- /dev/null +++ b/lib/gitlab/middleware/read_only.rb @@ -0,0 +1,88 @@ +module Gitlab + module Middleware + class ReadOnly + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + API_VERSIONS = (3..4) + + def initialize(app) + @app = app + @whitelisted = internal_routes + end + + def call(env) + @env = env + + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + error_message = 'You cannot do writing operations on a read-only GitLab instance' + + if json_request? + return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] + else + rack_flash.alert = error_message + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(env) + end + + private + + def internal_routes + API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } + end + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Rails.application.routes.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def logout_route + route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + request.path.end_with?('.git/git-upload-pack') + end + + def lfs_route + request.path.end_with?('/info/lfs/objects/batch') + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 68815be4d13..47c2a422387 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -64,6 +64,8 @@ module Gitlab protected def add_or_update_user_identities + return unless gl_user + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index e2fbcefdb74..cc37d708ce1 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -33,6 +33,7 @@ module Gitlab explore favicon.ico files + google_api groups health_check help diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 58f6245579a..bd677ec4bf3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -65,5 +65,9 @@ module Gitlab "can contain only lowercase letters, digits, and '-'. " \ "Must start with a letter, and cannot end with '-'" end + + def build_trace_section_regex + @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/.freeze + end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index a99f8e2b5f8..a37112ae5c4 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -222,10 +222,18 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) - path = full_path(storage, name) - FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).add(name) + else + path = full_path(storage, name) + FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + end + end rescue Errno::EEXIST => e Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Remove directory from repositories storage @@ -236,7 +244,15 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) - FileUtils.rm_r(full_path(storage, name), force: true) + Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).remove(name) + else + FileUtils.rm_r(full_path(storage, name), force: true) + end + end + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Move namespace directory inside repositories storage @@ -246,9 +262,17 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) - return false if exists?(storage, new_name) || !exists?(storage, old_name) + Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).rename(old_name, new_name) + else + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + end + end + rescue GRPC::InvalidArgument + false end def url_to_repo(path) @@ -272,7 +296,13 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - File.exist?(full_path(storage, dir_name)) + Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + if enabled + gitaly_namespace_client(storage).exists?(dir_name) + else + File.exist?(full_path(storage, dir_name)) + end + end end protected @@ -349,6 +379,14 @@ module Gitlab Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end + def gitaly_namespace_client(storage_path) + storage, _value = Gitlab.config.repositories.storages.find do |storage, value| + value['path'] == storage_path + end + + Gitlab::GitalyClient::NamespaceService.new(storage) + end + def gitaly_migrate(method, &block) Gitlab::GitalyClient.migrate(method, &block) rescue GRPC::NotFound, GRPC::BadStatus => e diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6857038dba8..3f3ba77d47f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -48,6 +48,7 @@ module Gitlab deploy_keys: DeployKey.count, deployments: Deployment.count, environments: ::Environment.count, + gcp_clusters: ::Gcp::Cluster.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index f200c694562..58d5b0da1c4 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -103,11 +103,16 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawDiffRequest' => Gitaly::RawDiffRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } + else + workhorse_diff_or_patch_hash(repository, diff_refs) + end [ SEND_DATA_HEADER, @@ -116,11 +121,16 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawPatchRequest' => Gitaly::RawPatchRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } + else + workhorse_diff_or_patch_hash(repository, diff_refs) + end [ SEND_DATA_HEADER, @@ -216,6 +226,22 @@ module Gitlab token: Gitlab::GitalyClient.token(repository.project.repository_storage) } end + + def workhorse_diff_or_patch_hash(repository, diff_refs) + { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => diff_refs.base_sha, + 'ShaTo' => diff_refs.head_sha + } + end + + def gitaly_diff_or_patch_hash(repository, diff_refs) + { + repository: repository.gitaly_repository, + left_commit_id: diff_refs.base_sha, + right_commit_id: diff_refs.head_sha + } + end end end end |