diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/entities.rb | 2 | ||||
| -rw-r--r-- | lib/api/groups.rb | 2 | ||||
| -rw-r--r-- | lib/api/helpers.rb | 8 | ||||
| -rw-r--r-- | lib/gitlab/fake_application_settings.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/git/object_pool.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client.rb | 42 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/object_pool_service.rb | 5 | ||||
| -rw-r--r-- | lib/gitlab/import_export/command_line_util.rb | 8 | ||||
| -rw-r--r-- | lib/gitlab/json_cache.rb | 87 | ||||
| -rw-r--r-- | lib/gitlab/object_hierarchy.rb (renamed from lib/gitlab/group_hierarchy.rb) | 52 | ||||
| -rw-r--r-- | lib/gitlab/upgrader.rb | 111 | ||||
| -rw-r--r-- | lib/gitlab/utils/override.rb | 109 | ||||
| -rw-r--r-- | lib/tasks/gitlab/storage.rake | 10 |
13 files changed, 258 insertions, 184 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index b83a5c14190..22403664c21 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -323,7 +323,7 @@ module API expose :request_access_enabled expose :full_name, :full_path - if ::Group.supports_nested_groups? + if ::Group.supports_nested_objects? expose :parent_id end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 626a2008dee..64958ff982a 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -113,7 +113,7 @@ module API requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - if ::Group.supports_nested_groups? + if ::Group.supports_nested_objects? optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a7d67a6300e..c3eca713712 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -163,9 +163,11 @@ module API end def find_branch!(branch_name) - user_project.repository.find_branch(branch_name) || not_found!('Branch') - rescue Gitlab::Git::CommandError - render_api_error!('The branch refname is invalid', 400) + if Gitlab::GitRefValidator.validate(branch_name) + user_project.repository.find_branch(branch_name) || not_found!('Branch') + else + render_api_error!('The branch refname is invalid', 400) + end end def find_project_label(id) diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index db1aeeea8d3..bd806269bf0 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -37,5 +37,9 @@ module Gitlab def pick_repository_storage repository_storages.sample end + + def commit_email_hostname + super.presence || ApplicationSetting.default_commit_email_hostname + end end end diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index ba9b40b7df4..1c6242b444a 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -8,7 +8,7 @@ module Gitlab GL_REPOSITORY = "" delegate :exists?, :size, to: :repository - delegate :delete, to: :object_pool_service + delegate :unlink_repository, :delete, to: :object_pool_service attr_reader :storage, :relative_path, :source_repository diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 11021ee06b3..8bf8a3b53cd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -26,6 +26,7 @@ module Gitlab end end + PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze @@ -50,11 +51,42 @@ module Gitlab @stubs[storage][name] ||= begin klass = stub_class(name) addr = stub_address(storage) - klass.new(addr, :this_channel_is_insecure) + creds = stub_creds(storage) + klass.new(addr, creds) end end end + def self.stub_cert_paths + cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"] + cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE + cert_paths + end + + def self.stub_certs + return @certs if @certs + + @certs = stub_cert_paths.flat_map do |cert_file| + File.read(cert_file).scan(PEM_REGEX).map do |cert| + begin + OpenSSL::X509::Certificate.new(cert).to_pem + rescue OpenSSL::OpenSSLError => e + Rails.logger.error "Could not load certificate #{cert_file} #{e}" + Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) + nil + end + end.compact + end.uniq.join("\n") + end + + def self.stub_creds(storage) + if URI(address(storage)).scheme == 'tls' + GRPC::Core::ChannelCredentials.new stub_certs + else + :this_channel_is_insecure + end + end + def self.stub_class(name) if name == :health_check Grpc::Health::V1::Health::Stub @@ -64,9 +96,7 @@ module Gitlab end def self.stub_address(storage) - addr = address(storage) - addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' - addr + address(storage).sub(%r{^tcp://|^tls://}, '') end def self.clear_stubs! @@ -88,8 +118,8 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless URI(address).scheme.in?(%w(tcp unix)) - raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" + unless URI(address).scheme.in?(%w(tcp unix tls)) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" end address diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index 272ce73ad64..6e7ede5fd18 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -35,7 +35,10 @@ module Gitlab end def unlink_repository(repository) - request = Gitaly::UnlinkRepositoryFromObjectPoolRequest.new(repository: repository.gitaly_repository) + request = Gitaly::UnlinkRepositoryFromObjectPoolRequest.new( + object_pool: object_pool, + repository: repository.gitaly_repository + ) GitalyClient.call(storage, :object_pool_service, :unlink_repository_from_object_pool, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index c9e2a6a78d9..bdecff0931c 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport module CommandLineUtil - DEFAULT_MODE = 0700 + UNTAR_MASK = 'u+rwX,go+rX,go-w' + DEFAULT_DIR_MODE = 0700 def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') @@ -14,8 +15,8 @@ module Gitlab end def mkdir_p(path) - FileUtils.mkdir_p(path, mode: DEFAULT_MODE) - FileUtils.chmod(DEFAULT_MODE, path) + FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) + FileUtils.chmod(DEFAULT_DIR_MODE, path) end private @@ -41,6 +42,7 @@ module Gitlab def untar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir})) + execute(%W(chmod -R #{UNTAR_MASK} #{dir})) end def execute(cmd) diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb new file mode 100644 index 00000000000..1adf83739ad --- /dev/null +++ b/lib/gitlab/json_cache.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + class JsonCache + attr_reader :backend, :cache_key_with_version, :namespace + + def initialize(options = {}) + @backend = options.fetch(:backend, Rails.cache) + @namespace = options.fetch(:namespace, nil) + @cache_key_with_version = options.fetch(:cache_key_with_version, true) + end + + def active? + if backend.respond_to?(:active?) + backend.active? + else + true + end + end + + def cache_key(key) + expanded_cache_key = [namespace, key].compact + + if cache_key_with_version + expanded_cache_key << Rails.version + end + + expanded_cache_key.join(':') + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def read(key, klass = nil) + value = backend.read(cache_key(key)) + value = parse_value(value, klass) if value + value + end + + def write(key, value, options = nil) + backend.write(cache_key(key), value.to_json, options) + end + + def fetch(key, options = {}, &block) + klass = options.delete(:as) + value = read(key, klass) + + return value unless value.nil? + + value = yield + + write(key, value, options) + + value + end + + private + + def parse_value(raw, klass) + value = ActiveSupport::JSON.decode(raw) + + case value + when Hash then parse_entry(value, klass) + when Array then parse_entries(value, klass) + else + value + end + rescue ActiveSupport::JSON.parse_error + nil + end + + def parse_entry(raw, klass) + klass.new(raw) if valid_entry?(raw, klass) + end + + def valid_entry?(raw, klass) + return false unless klass && raw.is_a?(Hash) + + (raw.keys - klass.attribute_names).empty? + end + + def parse_entries(values, klass) + values.map { |value| parse_entry(value, klass) }.compact + end + end +end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 97cbdc6cb39..f2772c733c7 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -1,16 +1,16 @@ # frozen_string_literal: true module Gitlab - # Retrieving of parent or child groups based on a base ActiveRecord relation. + # Retrieving of parent or child objects based on a base ActiveRecord relation. # # This class uses recursive CTEs and as a result will only work on PostgreSQL. - class GroupHierarchy + class ObjectHierarchy attr_reader :ancestors_base, :descendants_base, :model # ancestors_base - An instance of ActiveRecord::Relation for which to - # get parent groups. + # get parent objects. # descendants_base - An instance of ActiveRecord::Relation for which to - # get child groups. If omitted, ancestors_base is used. + # get child objects. If omitted, ancestors_base is used. def initialize(ancestors_base, descendants_base = ancestors_base) raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model @@ -39,7 +39,7 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Returns a relation that includes the ancestors_base set of groups + # Returns a relation that includes the ancestors_base set of objects # and all their ancestors (recursively). # # Passing an `upto` will stop the recursion once the specified parent_id is @@ -47,13 +47,13 @@ module Gitlab # included. # # Passing a `hierarchy_order` with either `:asc` or `:desc` will cause the - # recursive query order from most nested group to root or from the root - # ancestor to most nested group respectively. This uses a `depth` column + # recursive query order from most nested object to root or from the root + # ancestor to most nested object respectively. This uses a `depth` column # where `1` is defined as the depth for the base and increment as we go up # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless Group.supports_nested_groups? + return ancestors_base unless hierarchy_supported? recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -62,16 +62,16 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Returns a relation that includes the descendants_base set of groups + # Returns a relation that includes the descendants_base set of objects # and all their descendants (recursively). def base_and_descendants - return descendants_base unless Group.supports_nested_groups? + return descendants_base unless hierarchy_supported? read_only(base_and_descendants_cte.apply_to(model.all)) end - # Returns a relation that includes the base groups, their ancestors, - # and the descendants of the base groups. + # Returns a relation that includes the base objects, their ancestors, + # and the descendants of the base objects. # # The resulting query will roughly look like the following: # @@ -91,16 +91,16 @@ module Gitlab # Using this approach allows us to further add criteria to the relation with # Rails thinking it's selecting data the usual way. # - # If nested groups are not supported, ancestors_base is returned. + # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord - def all_groups - return ancestors_base unless Group.supports_nested_groups? + def all_objects + return ancestors_base unless hierarchy_supported? ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte - ancestors_table = ancestors.alias_to(groups_table) - descendants_table = descendants.alias_to(groups_table) + ancestors_table = ancestors.alias_to(objects_table) + descendants_table = descendants.alias_to(objects_table) relation = model .unscoped @@ -117,23 +117,27 @@ module Gitlab private + def hierarchy_supported? + Gitlab::Database.postgresql? + end + # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) depth_column = :depth base_query = ancestors_base.except(:order) - base_query = base_query.select("1 as #{depth_column}", groups_table[Arel.star]) if hierarchy_order + base_query = base_query.select("1 as #{depth_column}", objects_table[Arel.star]) if hierarchy_order cte << base_query # Recursively get all the ancestors of the base set. parent_query = model - .from([groups_table, cte.table]) - .where(groups_table[:id].eq(cte.table[:parent_id])) + .from([objects_table, cte.table]) + .where(objects_table[:id].eq(cte.table[:parent_id])) .except(:order) - parent_query = parent_query.select(cte.table[depth_column] + 1, groups_table[Arel.star]) if hierarchy_order + parent_query = parent_query.select(cte.table[depth_column] + 1, objects_table[Arel.star]) if hierarchy_order parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id cte << parent_query @@ -149,15 +153,15 @@ module Gitlab # Recursively get all the descendants of the base set. cte << model - .from([groups_table, cte.table]) - .where(groups_table[:parent_id].eq(cte.table[:id])) + .from([objects_table, cte.table]) + .where(objects_table[:parent_id].eq(cte.table[:id])) .except(:order) cte end # rubocop: enable CodeReuse/ActiveRecord - def groups_table + def objects_table model.arel_table end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb deleted file mode 100644 index ccab0e4dd73..00000000000 --- a/lib/gitlab/upgrader.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class Upgrader - def execute - puts "GitLab #{current_version.major} upgrade tool" - puts "Your version is #{current_version}" - puts "Latest available version for GitLab #{current_version.major} is #{latest_version}" - - if latest_version? - puts "You are using the latest GitLab version" - else - puts "Newer GitLab version is available" - - answer = if ARGV.first == "-y" - "yes" - else - prompt("Do you want to upgrade (yes/no)? ", %w{yes no}) - end - - if answer == "yes" - upgrade - else - exit 0 - end - end - end - - def latest_version? - current_version >= latest_version - end - - def current_version - @current_version ||= Gitlab::VersionInfo.parse(current_version_raw) - end - - def latest_version - @latest_version ||= Gitlab::VersionInfo.parse(latest_version_raw) - end - - def current_version_raw - File.read(File.join(gitlab_path, "VERSION")).strip - end - - def latest_version_raw - git_tags = fetch_git_tags - git_tags = git_tags.select { |version| version =~ /v\d+\.\d+\.\d+\Z/ } - git_versions = git_tags.map { |tag| Gitlab::VersionInfo.parse(tag.match(/v\d+\.\d+\.\d+/).to_s) } - "v#{git_versions.sort.last}" - end - - def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) - remote_tags.split("\n").grep(%r{tags/v#{current_version.major}}) - end - - def update_commands - { - "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), - "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), - "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), - "Install gems" => %w(bundle), - "Migrate DB" => %w(bundle exec rake db:migrate), - "Recompile assets" => %w(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile), - "Clear cache" => %w(bundle exec rake cache:clear) - } - end - - def env - { - 'RAILS_ENV' => 'production', - 'NODE_ENV' => 'production' - } - end - - def upgrade - update_commands.each do |title, cmd| - puts title - puts " -> #{cmd.join(' ')}" - - if system(env, *cmd) - puts " -> OK" - else - puts " -> FAILED" - puts "Failed to upgrade. Try to repeat task or proceed with upgrade manually " - exit 1 - end - end - - puts "Done" - end - - def gitlab_path - File.expand_path(File.join(File.dirname(__FILE__), '../..')) - end - - # Prompt the user to input something - # - # message - the message to display before input - # choices - array of strings of acceptable answers or nil for any answer - # - # Returns the user's answer - def prompt(message, choices = nil) - begin - print(message) - answer = STDIN.gets.chomp - end while !choices.include?(answer) - answer - end - end -end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index c412961ea3f..c87e97d0213 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -4,16 +4,11 @@ module Gitlab module Utils module Override class Extension - def self.verify_class!(klass, method_name) - instance_method_defined?(klass, method_name) || - raise( - NotImplementedError.new( - "#{klass}\##{method_name} doesn't exist!")) - end - - def self.instance_method_defined?(klass, name, include_super: true) - klass.instance_methods(include_super).include?(name) || - klass.private_instance_methods(include_super).include?(name) + def self.verify_class!(klass, method_name, arity) + extension = new(klass) + parents = extension.parents_for(klass) + extension.verify_method!( + klass: klass, parents: parents, method_name: method_name, sub_method_arity: arity) end attr_reader :subject @@ -22,35 +17,77 @@ module Gitlab @subject = subject end - def add_method_name(method_name) - method_names << method_name - end - - def add_class(klass) - classes << klass + def parents_for(klass) + index = klass.ancestors.index(subject) + klass.ancestors.drop(index + 1) end def verify! classes.each do |klass| - index = klass.ancestors.index(subject) - parents = klass.ancestors.drop(index + 1) - - method_names.each do |method_name| - parents.any? do |parent| - self.class.instance_method_defined?( - parent, method_name, include_super: false) - end || - raise( - NotImplementedError.new( - "#{klass}\##{method_name} doesn't exist!")) + parents = parents_for(klass) + + method_names.each_pair do |method_name, arity| + verify_method!( + klass: klass, + parents: parents, + method_name: method_name, + sub_method_arity: arity) end end end + def verify_method!(klass:, parents:, method_name:, sub_method_arity:) + overridden_parent = parents.find do |parent| + instance_method_defined?(parent, method_name) + end + + raise NotImplementedError.new("#{klass}\##{method_name} doesn't exist!") unless overridden_parent + + super_method_arity = find_direct_method(overridden_parent, method_name).arity + + unless arity_compatible?(sub_method_arity, super_method_arity) + raise NotImplementedError.new("#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}") + end + end + + def add_method_name(method_name, arity = nil) + method_names[method_name] = arity + end + + def add_class(klass) + classes << klass + end + + def verify_override?(method_name) + method_names.has_key?(method_name) + end + private + def instance_method_defined?(klass, name) + klass.instance_methods(false).include?(name) || + klass.private_instance_methods(false).include?(name) + end + + def find_direct_method(klass, name) + method = klass.instance_method(name) + method = method.super_method until method && klass == method.owner + method + end + + def arity_compatible?(sub_method_arity, super_method_arity) + if sub_method_arity >= 0 && super_method_arity >= 0 + # Regular arguments + sub_method_arity == super_method_arity + else + # It's too complex to check this case, just allow sub-method having negative arity + # But we don't allow sub_method_arity > 0 yet super_method_arity < 0 + sub_method_arity < 0 + end + end + def method_names - @method_names ||= [] + @method_names ||= {} end def classes @@ -80,11 +117,21 @@ module Gitlab def override(method_name) return unless ENV['STATIC_VERIFICATION'] + Override.extensions[self] ||= Extension.new(self) + Override.extensions[self].add_method_name(method_name) + end + + def method_added(method_name) + super + + return unless ENV['STATIC_VERIFICATION'] + return unless Override.extensions[self]&.verify_override?(method_name) + + method_arity = instance_method(method_name).arity if is_a?(Class) - Extension.verify_class!(self, method_name) + Extension.verify_class!(self, method_name, method_arity) else # We delay the check for modules - Override.extensions[self] ||= Extension.new(self) - Override.extensions[self].add_method_name(method_name) + Override.extensions[self].add_method_name(method_name, method_arity) end end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index f539b1df955..09dc3aa9882 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -2,6 +2,12 @@ namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do + if Gitlab::Database.read_only? + warn 'This task requires database write access. Exiting.' + + next + end + storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper @@ -9,7 +15,7 @@ namespace :gitlab do project = Project.with_unmigrated_storage.find_by(id: helper.range_from) unless project - puts "There are no projects requiring storage migration with ID=#{helper.range_from}" + warn "There are no projects requiring storage migration with ID=#{helper.range_from}" next end @@ -23,7 +29,7 @@ namespace :gitlab do legacy_projects_count = Project.with_unmigrated_storage.count if legacy_projects_count == 0 - puts 'There are no projects requiring storage migration. Nothing to do!' + warn 'There are no projects requiring storage migration. Nothing to do!' next end |