diff options
Diffstat (limited to 'lib/gitlab')
51 files changed, 571 insertions, 172 deletions
diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index c62aeb60fa9..b88b2e36d53 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -2,7 +2,7 @@ module Gitlab module Ci module Build class Image - attr_reader :name + attr_reader :alias, :command, :entrypoint, :name class << self def from_image(job) @@ -21,7 +21,14 @@ module Gitlab end def initialize(image) - @name = image + if image.is_a?(String) + @name = image + elsif image.is_a?(Hash) + @alias = image[:alias] + @command = image[:command] + @entrypoint = image[:entrypoint] + @name = image[:name] + end end def valid? diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index b5050257688..897dcff8012 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -8,8 +8,36 @@ module Gitlab class Image < Node include Validatable + ALLOWED_KEYS = %i[name entrypoint].freeze + validations do - validates :config, type: String + validates :config, hash_or_string: true + validates :config, allowed_keys: ALLOWED_KEYS + + validates :name, type: String, presence: true + validates :entrypoint, type: String, allow_nil: true + end + + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def name + value[:name] + end + + def entrypoint + value[:entrypoint] + end + + def value + return { name: @config } if string? + return @config if hash? + {} end end end diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb new file mode 100644 index 00000000000..b52faf48b58 --- /dev/null +++ b/lib/gitlab/ci/config/entry/service.rb @@ -0,0 +1,34 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of Docker service. + # + class Service < Image + include Validatable + + ALLOWED_KEYS = %i[name entrypoint command alias].freeze + + validations do + validates :config, hash_or_string: true + validates :config, allowed_keys: ALLOWED_KEYS + + validates :name, type: String, presence: true + validates :entrypoint, type: String, allow_nil: true + validates :command, type: String, allow_nil: true + validates :alias, type: String, allow_nil: true + end + + def alias + value[:alias] + end + + def command + value[:command] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 84f8ab780f5..0066894e069 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -9,7 +9,30 @@ module Gitlab include Validatable validations do - validates :config, array_of_strings: true + validates :config, type: Array + end + + def compose!(deps = nil) + super do + @entries = [] + @config.each do |config| + @entries << Entry::Factory.new(Entry::Service) + .value(config || {}) + .create! + end + + @entries.each do |entry| + entry.compose!(deps) + end + end + end + + def value + @entries.map(&:value) + end + + def descendants + @entries end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index bd7428b1272..b2ca3c881e4 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -44,6 +44,14 @@ module Gitlab end end + class HashOrStringValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(String) + record.errors.add(attribute, 'should be a hash or a string') + end + end + end + class KeyValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb index 4969a350862..9307545b5b1 100644 --- a/lib/gitlab/ci/status/external/common.rb +++ b/lib/gitlab/ci/status/external/common.rb @@ -3,6 +3,10 @@ module Gitlab module Status module External module Common + def label + subject.description + end + def has_details? subject.target_url.present? && can?(user, :read_commit_status, subject) diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 48735fd197d..284e6ad55a5 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -10,43 +10,49 @@ module Gitlab delegate :sidekiq_throttling_enabled?, to: :current_application_settings - def fake_application_settings - OpenStruct.new(::ApplicationSetting.defaults) + def fake_application_settings(defaults = ApplicationSetting.defaults) + FakeApplicationSettings.new(defaults) end private def ensure_application_settings! - unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - settings = retrieve_settings_from_database? - end + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - settings || in_memory_application_settings + cached_application_settings || uncached_application_settings end - def retrieve_settings_from_database? - settings = retrieve_settings_from_database_cache? - return settings if settings.present? - - return fake_application_settings unless connect_to_db? - + def cached_application_settings begin - db_settings = ::ApplicationSetting.current - # In case Redis isn't running or the Redis UNIX socket file is not available + ApplicationSetting.cached rescue ::Redis::BaseError, ::Errno::ENOENT - db_settings = ::ApplicationSetting.last + # In case Redis isn't running or the Redis UNIX socket file is not available end - db_settings || ::ApplicationSetting.create_from_defaults end - def retrieve_settings_from_database_cache? + def uncached_application_settings + return fake_application_settings unless connect_to_db? + + # This loads from the database into the cache, so handle Redis errors begin - settings = ApplicationSetting.cached + db_settings = ApplicationSetting.current rescue ::Redis::BaseError, ::Errno::ENOENT # In case Redis isn't running or the Redis UNIX socket file is not available - settings = nil end - settings + + # If there are pending migrations, it's possible there are columns that + # need to be added to the application settings. To prevent Rake tasks + # and other callers from failing, use any loaded settings and return + # defaults for missing columns. + if ActiveRecord::Migrator.needs_migration? + defaults = ApplicationSetting.defaults + defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? + return fake_application_settings(defaults) + end + + return db_settings if db_settings.present? + + ApplicationSetting.create_from_defaults || in_memory_application_settings end def in_memory_application_settings @@ -62,8 +68,7 @@ module Gitlab active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') && - !ActiveRecord::Migrator.needs_migration? + ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d0bd1299671..0d5a7cf0694 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -83,6 +83,22 @@ module Gitlab end end + def self.bulk_insert(table, rows) + return if rows.empty? + + keys = rows.first.keys + columns = keys.map { |key| connection.quote_column_name(key) } + + tuples = rows.map do |row| + row.values_at(*keys).map { |value| connection.quote(value) } + end + + connection.execute <<-EOF.strip_heredoc + INSERT INTO #{table} (#{columns.join(', ')}) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + EOF + end + # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. def self.create_connection_pool(pool_size, host = nil) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index a412bb6dbd2..9181202a091 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1,6 +1,39 @@ module Gitlab module Database module MigrationHelpers + # Adds `created_at` and `updated_at` columns with timezone information. + # + # This method is an improved version of Rails' built-in method `add_timestamps`. + # + # Available options are: + # default - The default value for the column. + # null - When set to `true` the column will allow NULL values. + # The default is to not allow NULL values. + def add_timestamps_with_timezone(table_name, options = {}) + options[:null] = false if options[:null].nil? + + [:created_at, :updated_at].each do |column_name| + if options[:default] && transaction_open? + raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \ + 'You can disable transactions by calling `disable_ddl_transaction!` ' \ + 'in the body of your migration class' + end + + # If default value is presented, use `add_column_with_default` method instead. + if options[:default] + add_column_with_default( + table_name, + column_name, + :datetime_with_timezone, + default: options[:default], + allow_null: options[:null] + ) + else + add_column(table_name, column_name, :datetime_with_timezone, options) + end + end + end + # Creates a new index, concurrently when supported # # On PostgreSQL this method creates an index concurrently, on MySQL this @@ -200,6 +233,12 @@ module Gitlab # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil + max_size = 1000 + + # The upper limit is 1000 to ensure we don't lock too many rows. For + # example, for "merge_requests" even 1% of the table is around 35 000 + # rows for GitLab.com. + batch_size = max_size if batch_size > max_size start_arel = table.project(table[:id]).order(table[:id].asc).take(1) start_arel = yield table, start_arel if block_given? diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 4212a0dbe2e..d2863a4da71 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -5,7 +5,20 @@ module Gitlab delegate :new_file?, :deleted_file?, :renamed_file?, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, - :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false + :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, to: :diff, prefix: false + + # Finding a viewer for a diff file happens based only on extension and whether the + # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer, + # and the order of these viewers doesn't really matter. + # + # However, when the diff file blobs are LFS pointers, we cannot know for sure whether the + # file being pointed to is binary or text. In this case, we match only on + # extension, preferring binary viewers over text ones if both exist, since the + # large files referred to in "Large File Storage" are much more likely to be + # binary than text. + RICH_VIEWERS = [ + DiffViewer::Image + ].sort_by { |v| v.binary? ? 0 : 1 }.freeze def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) @diff = diff @@ -177,6 +190,100 @@ module Gitlab def text? !binary? end + + def external_storage_error? + old_blob&.external_storage_error? || new_blob&.external_storage_error? + end + + def stored_externally? + old_blob&.stored_externally? || new_blob&.stored_externally? + end + + def external_storage + old_blob&.external_storage || new_blob&.external_storage + end + + def content_changed? + old_blob && new_blob && old_blob.id != new_blob.id + end + + def different_type? + old_blob && new_blob && old_blob.binary? != new_blob.binary? + end + + def size + [old_blob&.size, new_blob&.size].compact.sum + end + + def raw_size + [old_blob&.raw_size, new_blob&.raw_size].compact.sum + end + + def raw_binary? + old_blob&.raw_binary? || new_blob&.raw_binary? + end + + def raw_text? + !raw_binary? && !different_type? + end + + def simple_viewer + @simple_viewer ||= simple_viewer_class.new(self) + end + + def rich_viewer + return @rich_viewer if defined?(@rich_viewer) + + @rich_viewer = rich_viewer_class&.new(self) + end + + def rendered_as_text?(ignore_errors: true) + simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?) + end + + private + + def simple_viewer_class + return DiffViewer::NotDiffable unless diffable? + + if content_changed? + if raw_text? + DiffViewer::Text + else + DiffViewer::NoPreview + end + elsif new_file? + if raw_text? + DiffViewer::Text + else + DiffViewer::Added + end + elsif deleted_file? + if raw_text? + DiffViewer::Text + else + DiffViewer::Deleted + end + elsif renamed_file? + DiffViewer::Renamed + elsif mode_changed? + DiffViewer::ModeChanged + end + end + + def rich_viewer_class + viewer_class_from(RICH_VIEWERS) + end + + def viewer_class_from(classes) + return unless diffable? + return if different_type? || external_storage_error? + return unless new_file? || deleted_file? || content_changed? + + verify_binary = !stored_externally? + + classes.find { |viewer_class| viewer_class.can_render?(self, verify_binary: verify_binary) } + end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index bd52ae47e9f..2d89ccfc354 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -42,25 +42,25 @@ module Gitlab end def added? - type == 'new' || type == 'new-nonewline' + %w[new new-nonewline].include?(type) end def removed? - type == 'old' || type == 'old-nonewline' - end - - def rich_text - @parent_file.highlight_lines! if @parent_file && !@rich_text - - @rich_text + %w[old old-nonewline].include?(type) end def meta? - type == 'match' + %w[match new-nonewline old-nonewline].include?(type) end def discussable? - !['match', 'new-nonewline', 'old-nonewline'].include?(type) + !meta? + end + + def rich_text + @parent_file.highlight_lines! if @parent_file && !@rich_text + + @rich_text end def as_json(opts = nil) diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 481536a380b..0cb26fa45c8 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -14,16 +14,7 @@ module Gitlab lines = [] highlighted_diff_lines = diff_file.highlighted_diff_lines highlighted_diff_lines.each do |line| - if line.meta? || line.unchanged? - # line in the right panel is the same as in the left one - lines << { - left: line, - right: line - } - - free_right_index = nil - i += 1 - elsif line.removed? + if line.removed? lines << { left: line, right: nil @@ -51,6 +42,15 @@ module Gitlab free_right_index = nil i += 1 end + elsif line.meta? || line.unchanged? + # line in the right panel is the same as in the left one + lines << { + left: line, + right: line + } + + free_right_index = nil + i += 1 end end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 7f884183bb1..1d6f5bb5e1c 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -7,7 +7,7 @@ module Gitlab def call(env) request = Rack::Request.new(env) - route = Gitlab::EtagCaching::Router.match(request) + route = Gitlab::EtagCaching::Router.match(request.path_info) return @app.call(env) unless route track_event(:etag_caching_middleware_used, route) diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index dccc66b3918..75167a6b088 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -53,8 +53,8 @@ module Gitlab ) ].freeze - def self.match(request) - ROUTES.find { |route| route.regexp.match(request.path_info) } + def self.match(path) + ROUTES.find { |route| route.regexp.match(path) } end end end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 0039fc01c8f..072fcfc65e6 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -25,6 +25,8 @@ module Gitlab end def redis_key(key) + raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) + "#{REDIS_NAMESPACE}#{key}" end end diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb new file mode 100644 index 00000000000..bb14a8cd9e7 --- /dev/null +++ b/lib/gitlab/fake_application_settings.rb @@ -0,0 +1,27 @@ +# This class extends an OpenStruct object by adding predicate methods to mimic +# ActiveRecord access. We rely on the initial values being true or false to +# determine whether to define a predicate method because for a newly-added +# column that has not been migrated yet, there is no way to determine the +# column type without parsing db/schema.rb. +module Gitlab + class FakeApplicationSettings < OpenStruct + def initialize(options = {}) + super + + FakeApplicationSettings.define_predicate_methods(options) + end + + # Mimic ActiveRecord predicate methods for boolean values + def self.define_predicate_methods(options) + options.each do |key, value| + next if key.to_s.end_with?('?') + next unless [true, false].include?(value) + + define_method "#{key}?" do + actual_key = key.to_s.chomp('?') + self[actual_key] + end + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 88ad760bea3..f825568f194 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -16,9 +16,11 @@ module Gitlab alias_method :renamed_file?, :renamed_file attr_accessor :expanded + attr_writer :too_large - # We need this accessor because of `to_hash` and `init_from_hash` - attr_accessor :too_large + alias_method :expanded?, :expanded + + SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze class << self # The maximum size of a diff to display. @@ -229,16 +231,10 @@ module Gitlab end end - def serialize_keys - @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large) - end - def to_hash hash = {} - keys = serialize_keys - - keys.each do |key| + SERIALIZE_KEYS.each do |key| hash[key] = send(key) end @@ -265,6 +261,9 @@ module Gitlab end end + # This is used by `to_hash` and `init_from_hash`. + alias_method :too_large, :too_large? + def too_large! @diff = '' @line_count = 0 @@ -313,7 +312,7 @@ module Gitlab def init_from_hash(hash) raw_diff = hash.symbolize_keys - serialize_keys.each do |key| + SERIALIZE_KEYS.each do |key| send(:"#{key}=", raw_diff[key.to_sym]) end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 334e06a6eca..555894907cc 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -97,7 +97,7 @@ module Gitlab diff = Gitlab::Git::Diff.new(raw, expanded: expanded) - if !expanded && over_safe_limits?(i) + if !expanded && over_safe_limits?(i) && diff.line_count > 0 diff.collapse! end diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb new file mode 100644 index 00000000000..f4e3b5e5129 --- /dev/null +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -0,0 +1,77 @@ +module Gitlab + module Git + class GitmodulesParser + def initialize(content) + @content = content + end + + # Parses the contents of a .gitmodules file and returns a hash of + # submodule information, indexed by path. + def parse + reindex_by_path(get_submodules_by_name) + end + + private + + class State + def initialize + @result = {} + @current_submodule = nil + end + + def start_section(section) + # In some .gitmodules files (e.g. nodegit's), a header + # with the same name appears multiple times; we want to + # accumulate the configs across these + @current_submodule = @result[section] || { 'name' => section } + @result[section] = @current_submodule + end + + def set_attribute(attr, value) + @current_submodule[attr] = value + end + + def section_started? + !@current_submodule.nil? + end + + def submodules_by_name + @result + end + end + + def get_submodules_by_name + iterator = State.new + + @content.split("\n").each_with_object(iterator) do |text, iterator| + next if text =~ /^\s*#/ + + if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/ + iterator.start_section($~[:name]) + else + next unless iterator.section_started? + + next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/ + + value = $~[:value].chomp + iterator.set_attribute($~[:key], value) + end + end + + iterator.submodules_by_name + end + + def reindex_by_path(submodules_by_name) + # Convert from an indexed by name to an array indexed by path + # If a submodule doesn't have a path, it is considered bogus + # and is ignored + submodules_by_name.each_with_object({}) do |(name, data), results| + path = data.delete 'path' + next unless path + + results[path] = data + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 85695d0a4df..c1f942f931a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -617,9 +617,9 @@ module Gitlab # # Ex. # { - # "rack" => { + # "current_path/rack" => { + # "name" => "original_path/rack", # "id" => "c67be4624545b4263184c4a0e8f887efd0a66320", - # "path" => "rack", # "url" => "git://github.com/chneukirchen/rack.git" # }, # "encoding" => { @@ -637,7 +637,8 @@ module Gitlab return {} end - parse_gitmodules(commit, content) + parser = GitmodulesParser.new(content) + fill_submodule_ids(commit, parser.parse) end # Return total commits count accessible from passed ref @@ -998,42 +999,19 @@ module Gitlab end end - # Parses the contents of a .gitmodules file and returns a hash of - # submodule information. - def parse_gitmodules(commit, content) - modules = {} - - name = nil - content.each_line do |line| - case line.strip - when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header - name = $~[:name] - modules[name] = {} - when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair - key = $~[:key] - value = $~[:value].chomp - - next unless name && modules[name] - - modules[name][key] = value - - if key == 'path' - begin - modules[name]['id'] = blob_content(commit, value) - rescue InvalidBlobName - # The current entry is invalid - modules.delete(name) - name = nil - end - end - when /\A#/ # Comment - next - else # Invalid line - name = nil + # Fill in the 'id' field of a submodule hash from its values + # as-of +commit+. Return a Hash consisting only of entries + # from the submodule hash for which the 'id' field is filled. + def fill_submodule_ids(commit, submodule_data) + submodule_data.each do |path, data| + id = begin + blob_content(commit, path) + rescue InvalidBlobName + nil end + data['id'] = id end - - modules + submodule_data.select { |path, data| data['id'] } end # Returns true if +commit+ introduced changes to +path+, using commit diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 0a19d24eb20..0b62911958d 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -22,12 +22,13 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities + attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path - def initialize(actor, project, protocol, authentication_abilities:) + def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil) @actor = actor @project = project @protocol = protocol + @redirected_path = redirected_path @authentication_abilities = authentication_abilities end @@ -35,6 +36,7 @@ module Gitlab check_protocol! check_active_user! check_project_accessibility! + check_project_moved! check_command_disabled!(cmd) check_command_existence!(cmd) check_repository_existence! @@ -87,6 +89,21 @@ module Gitlab end end + def check_project_moved! + if redirected_path + url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + message = <<-MESSAGE.strip_heredoc + Project '#{redirected_path}' was moved to '#{project.full_path}'. + + Please update your Git remote and try again: + + git remote set-url origin #{url} + MESSAGE + + raise NotFoundError, message + end + end + def check_command_disabled!(cmd) if upload_pack?(cmd) check_upload_pack_disabled! diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 86d055d3533..f5a4c5493ef 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -4,7 +4,6 @@ module Gitlab class << self def repository(repository_storage, relative_path) Gitaly::Repository.new( - path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path), storage_name: repository_storage, relative_path: relative_path ) diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index e9d5d52cabb..357c076e874 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -3,33 +3,38 @@ module Gitlab # # This class uses recursive CTEs and as a result will only work on PostgreSQL. class GroupHierarchy - attr_reader :base, :model - - # base - An instance of ActiveRecord::Relation for which to get parent or - # child groups. - def initialize(base) - @base = base - @model = base.model + attr_reader :ancestors_base, :descendants_base, :model + + # ancestors_base - An instance of ActiveRecord::Relation for which to + # get parent groups. + # descendants_base - An instance of ActiveRecord::Relation for which to + # get child groups. If omitted, ancestors_base is used. + def initialize(ancestors_base, descendants_base = ancestors_base) + raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model + + @ancestors_base = ancestors_base + @descendants_base = descendants_base + @model = ancestors_base.model end - # Returns a relation that includes the base set of groups and all their - # ancestors (recursively). + # Returns a relation that includes the ancestors_base set of groups + # and all their ancestors (recursively). def base_and_ancestors - return model.none unless Group.supports_nested_groups? + return ancestors_base unless Group.supports_nested_groups? base_and_ancestors_cte.apply_to(model.all) end - # Returns a relation that includes the base set of groups and all their - # descendants (recursively). + # Returns a relation that includes the descendants_base set of groups + # and all their descendants (recursively). def base_and_descendants - return model.none unless Group.supports_nested_groups? + return descendants_base unless Group.supports_nested_groups? base_and_descendants_cte.apply_to(model.all) end - # Returns a relation that includes the base groups, their ancestors, and the - # descendants of the base groups. + # Returns a relation that includes the base groups, their ancestors, + # and the descendants of the base groups. # # The resulting query will roughly look like the following: # @@ -48,8 +53,10 @@ module Gitlab # # Using this approach allows us to further add criteria to the relation with # Rails thinking it's selecting data the usual way. + # + # If nested groups are not supported, ancestors_base is returned. def all_groups - return base unless Group.supports_nested_groups? + return ancestors_base unless Group.supports_nested_groups? ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -72,7 +79,7 @@ module Gitlab def base_and_ancestors_cte cte = SQL::RecursiveCTE.new(:base_and_ancestors) - cte << base.except(:order) + cte << ancestors_base.except(:order) # Recursively get all the ancestors of the base set. cte << model. @@ -86,7 +93,7 @@ module Gitlab def base_and_descendants_cte cte = SQL::RecursiveCTE.new(:base_and_descendants) - cte << base.except(:order) + cte << descendants_base.except(:order) # Recursively get all the descendants of the base set. cte << model. diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 328dd17e452..a5ad2f952d3 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -6,6 +6,7 @@ module Gitlab 'en' => 'English', 'es' => 'Español', 'de' => 'Deutsch', + 'fr' => 'Français', 'pt_BR' => 'Português(Brasil)', 'zh_CN' => '简体中文', 'zh_HK' => '繁體中文(香港)', diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 27d5a9198b6..3470a09eaf0 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.7'.freeze + VERSION = '0.1.8'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ff2b1d08c3c..72183e8aad4 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -26,7 +26,8 @@ project_tree: - notes: - :author - :events - - :merge_request_diff + - merge_request_diff: + - :merge_request_diff_files - :events - :timelogs - label_links: @@ -92,6 +93,8 @@ excluded_attributes: - :expired_at merge_request_diff: - :st_diffs + merge_request_diff_files: + - :diff issues: - :milestone_id merge_requests: @@ -113,6 +116,8 @@ methods: - :type merge_request_diff: - :utf8_st_diffs + merge_request_diff_files: + - :utf8_diff merge_requests: - :diff_head_sha project: diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 48c09dafcb6..b48f63bcd7e 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -83,7 +83,9 @@ module Gitlab # +value+ existing model to be included in the hash # +json_config_hash+ the original hash containing the root model def add_model_value(current_key, value, json_config_hash) - @attributes_finder.parse(value) { |hash| value = { value => hash } } + @attributes_finder.parse(value) do |hash| + value = { value => hash } unless value.is_a?(Hash) + end add_to_array(current_key, json_config_hash, value) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 695852526cb..20580459046 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -71,6 +71,7 @@ module Gitlab @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diff_commits if @relation_name == :merge_request_diff + set_diff if @relation_name == :merge_request_diff_files end def update_user_references @@ -202,6 +203,10 @@ module Gitlab HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) end + def set_diff + @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 8db91d25a4b..208f0e1bbea 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -14,7 +14,7 @@ module Gitlab # timeout - The maximum amount of seconds to block the caller for. This # ensures we don't indefinitely block a caller in case a job takes # long to process, or is never processed. - def wait(timeout = 60) + def wait(timeout = 10) start = Time.current while (Time.current - start) <= timeout diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 4a6091488c8..c56c1a4322f 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -8,13 +8,13 @@ module Gitlab ) # Filters an array of pods (as returned by the kubernetes API) by their labels - def filter_pods(pods, labels = {}) - pods.select do |pod| - metadata = pod.fetch("metadata", {}) - pod_labels = metadata.fetch("labels", nil) - next unless pod_labels + def filter_by_label(items, labels = {}) + items.select do |item| + metadata = item.fetch("metadata", {}) + item_labels = metadata.fetch("labels", nil) + next unless item_labels - labels.all? { |k, v| pod_labels[k.to_s] == v } + labels.all? { |k, v| item_labels[k.to_s] == v } end end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index caab8856014..3937d9c153a 100644 --- a/lib/gitlab/slash_commands/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -1,5 +1,5 @@ module Gitlab - module SlashCommands + module QuickActions class CommandDefinition attr_accessor :name, :aliases, :description, :explanation, :params, :condition_block, :parse_params_block, :action_block diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index 1b5b4566d81..a4a97236ffc 100644 --- a/lib/gitlab/slash_commands/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -1,5 +1,5 @@ module Gitlab - module SlashCommands + module QuickActions module Dsl extend ActiveSupport::Concern @@ -14,7 +14,7 @@ module Gitlab end class_methods do - # Allows to give a description to the next slash command. + # Allows to give a description to the next quick action. # This description is shown in the autocomplete menu. # It accepts a block that will be evaluated with the context given to # `CommandDefintion#to_h`. @@ -31,7 +31,7 @@ module Gitlab @description = block_given? ? block : text end - # Allows to define params for the next slash command. + # Allows to define params for the next quick action. # These params are shown in the autocomplete menu. # # Example: diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 6dbb467d70d..09576be7156 100644 --- a/lib/gitlab/slash_commands/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -1,10 +1,10 @@ module Gitlab - module SlashCommands + module QuickActions # This class takes an array of commands that should be extracted from a # given text. # # ``` - # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels]) # ``` class Extractor attr_reader :command_definitions @@ -24,7 +24,7 @@ module Gitlab # # Usage: # ``` - # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels]) # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 878e03f61d7..3591fa9145e 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -3,16 +3,18 @@ module Gitlab NotFoundError = Class.new(StandardError) def self.parse(repo_path) + wiki = false project_path = strip_storage_path(repo_path.sub(/\.git\z/, ''), fail_on_not_found: false) - project = Project.find_by_full_path(project_path) - if project_path.end_with?('.wiki') && !project - project = Project.find_by_full_path(project_path.chomp('.wiki')) + project, was_redirected = find_project(project_path) + + if project_path.end_with?('.wiki') && project.nil? + project, was_redirected = find_project(project_path.chomp('.wiki')) wiki = true - else - wiki = false end - [project, wiki] + redirected_path = project_path if was_redirected + + [project, wiki, redirected_path] end def self.strip_storage_path(repo_path, fail_on_not_found: true) @@ -30,5 +32,12 @@ module Gitlab result.sub(/\A\/*/, '') end + + def self.find_project(project_path) + project = Project.find_by_full_path(project_path, follow_redirects: true) + was_redirected = project && project.full_path.casecmp(project_path) != 0 + + [project, was_redirected] + end end end diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index 25da8474e95..cc3c9a50555 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class BaseCommand QUERY_LIMIT = 5 diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 3e0c30c33b7..a78408b0519 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -1,11 +1,11 @@ module Gitlab - module ChatCommands + module SlashCommands class Command < BaseCommand COMMANDS = [ - Gitlab::ChatCommands::IssueShow, - Gitlab::ChatCommands::IssueNew, - Gitlab::ChatCommands::IssueSearch, - Gitlab::ChatCommands::Deploy + Gitlab::SlashCommands::IssueShow, + Gitlab::SlashCommands::IssueNew, + Gitlab::SlashCommands::IssueSearch, + Gitlab::SlashCommands::Deploy ].freeze def execute @@ -15,10 +15,10 @@ module Gitlab if command.allowed?(project, current_user) command.new(project, current_user, params).execute(match) else - Gitlab::ChatCommands::Presenters::Access.new.access_denied + Gitlab::SlashCommands::Presenters::Access.new.access_denied end else - Gitlab::ChatCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) + Gitlab::SlashCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) end end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 458d90f84e8..e71eb15d604 100644 --- a/lib/gitlab/chat_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class Deploy < BaseCommand def self.match(text) /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) @@ -24,12 +24,12 @@ module Gitlab actions = find_actions(from, to) if actions.none? - Gitlab::ChatCommands::Presenters::Deploy.new(nil).no_actions + Gitlab::SlashCommands::Presenters::Deploy.new(nil).no_actions elsif actions.one? action = play!(from, to, actions.first) - Gitlab::ChatCommands::Presenters::Deploy.new(action).present(from, to) + Gitlab::SlashCommands::Presenters::Deploy.new(action).present(from, to) else - Gitlab::ChatCommands::Presenters::Deploy.new(actions).too_many_actions + Gitlab::SlashCommands::Presenters::Deploy.new(actions).too_many_actions end end diff --git a/lib/gitlab/chat_commands/help.rb b/lib/gitlab/slash_commands/help.rb index 6c0e4d304a4..81f3707e03e 100644 --- a/lib/gitlab/chat_commands/help.rb +++ b/lib/gitlab/slash_commands/help.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class Help < BaseCommand # This class has to be used last, as it always matches. It has to match # because other commands were not triggered and we want to show the help @@ -17,7 +17,7 @@ module Gitlab end def execute(commands, text) - Gitlab::ChatCommands::Presenters::Help.new(commands).present(trigger, text) + Gitlab::SlashCommands::Presenters::Help.new(commands).present(trigger, text) end def trigger diff --git a/lib/gitlab/chat_commands/issue_command.rb b/lib/gitlab/slash_commands/issue_command.rb index 84de3e44c70..87ea19b8806 100644 --- a/lib/gitlab/chat_commands/issue_command.rb +++ b/lib/gitlab/slash_commands/issue_command.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueCommand < BaseCommand def self.available?(project) project.issues_enabled? && project.default_issues_tracker? diff --git a/lib/gitlab/chat_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 016054ecd46..25f965e843d 100644 --- a/lib/gitlab/chat_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueNew < IssueCommand def self.match(text) # we can not match \n with the dot by passing the m modifier as than @@ -35,7 +35,7 @@ module Gitlab end def presenter(issue) - Gitlab::ChatCommands::Presenters::IssueNew.new(issue) + Gitlab::SlashCommands::Presenters::IssueNew.new(issue) end end end diff --git a/lib/gitlab/chat_commands/issue_search.rb b/lib/gitlab/slash_commands/issue_search.rb index 3491b53093e..acba84b54b4 100644 --- a/lib/gitlab/chat_commands/issue_search.rb +++ b/lib/gitlab/slash_commands/issue_search.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueSearch < IssueCommand def self.match(text) /\Aissue\s+search\s+(?<query>.*)/.match(text) diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/slash_commands/issue_show.rb index d6013f4d10c..ffa5184e5cb 100644 --- a/lib/gitlab/chat_commands/issue_show.rb +++ b/lib/gitlab/slash_commands/issue_show.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueShow < IssueCommand def self.match(text) /\Aissue\s+show\s+#{Issue.reference_prefix}?(?<iid>\d+)/.match(text) @@ -13,9 +13,9 @@ module Gitlab issue = find_by_iid(match[:iid]) if issue - Gitlab::ChatCommands::Presenters::IssueShow.new(issue).present + Gitlab::SlashCommands::Presenters::IssueShow.new(issue).present else - Gitlab::ChatCommands::Presenters::Access.new.not_found + Gitlab::SlashCommands::Presenters::Access.new.not_found end end end diff --git a/lib/gitlab/chat_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index 92f4fa17f78..1a817eb735b 100644 --- a/lib/gitlab/chat_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class Access < Presenters::Base def access_denied diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 05994bee79d..27696436574 100644 --- a/lib/gitlab/chat_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class Base include Gitlab::Routing.url_helpers diff --git a/lib/gitlab/chat_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb index 863d0bf99ca..b8dc77bd37b 100644 --- a/lib/gitlab/chat_commands/presenters/deploy.rb +++ b/lib/gitlab/slash_commands/presenters/deploy.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class Deploy < Presenters::Base def present(from, to) diff --git a/lib/gitlab/chat_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index cd47b7f4c6a..ea611a4d629 100644 --- a/lib/gitlab/chat_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class Help < Presenters::Base def present(trigger, text) diff --git a/lib/gitlab/chat_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index 25bc82994ba..341f2aabdd0 100644 --- a/lib/gitlab/chat_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters module IssueBase def color(issuable) diff --git a/lib/gitlab/chat_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index 3674ba25641..86490a39cc1 100644 --- a/lib/gitlab/chat_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class IssueNew < Presenters::Base include Presenters::IssueBase diff --git a/lib/gitlab/chat_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb index 73788cf9662..4e27d668685 100644 --- a/lib/gitlab/chat_commands/presenters/issue_search.rb +++ b/lib/gitlab/slash_commands/presenters/issue_search.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class IssueSearch < Presenters::Base include Presenters::IssueBase diff --git a/lib/gitlab/chat_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb index bd784ad241e..c99316df667 100644 --- a/lib/gitlab/chat_commands/presenters/issue_show.rb +++ b/lib/gitlab/slash_commands/presenters/issue_show.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class IssueShow < Presenters::Base include Presenters::IssueBase diff --git a/lib/gitlab/chat_commands/result.rb b/lib/gitlab/slash_commands/result.rb index 324d7ef43a3..7021b4b01b2 100644 --- a/lib/gitlab/chat_commands/result.rb +++ b/lib/gitlab/slash_commands/result.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands Result = Struct.new(:type, :message) end end |