diff options
Diffstat (limited to 'lib/gitlab')
66 files changed, 1659 insertions, 508 deletions
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 96d38f6daa0..3d41ac76406 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -20,21 +20,20 @@ module Gitlab backend: :gitlab_html5, attributes: DEFAULT_ADOC_ATTRS } - context[:pipeline] = :markup + context[:pipeline] = :ascii_doc plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) html = Banzai.render(html, context) - html.html_safe end def self.plantuml_setup Asciidoctor::PlantUml.configure do |conf| - conf.url = ApplicationSetting.current.plantuml_url - conf.svg_enable = ApplicationSetting.current.plantuml_enabled - conf.png_enable = ApplicationSetting.current.plantuml_enabled + conf.url = current_application_settings.plantuml_url + conf.svg_enable = current_application_settings.plantuml_enabled + conf.png_enable = current_application_settings.plantuml_enabled conf.txt_enable = false end end @@ -47,13 +46,13 @@ module Gitlab def stem(node) return super unless node.style.to_sym == :latexmath - %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>) + %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>) end def inline_quoted(node) return super unless node.type.to_sym == :latexmath - %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>) + %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>) end private diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb index 2700a5a2ad5..05994bee79d 100644 --- a/lib/gitlab/chat_commands/presenters/base.rb +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -45,9 +45,9 @@ module Gitlab end def format_response(response) - response[:text] = format(response[:text]) if response.has_key?(:text) + response[:text] = format(response[:text]) if response.key?(:text) - if response.has_key?(:attachments) + if response.key?(:attachments) response[:attachments].each do |attachment| attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] attachment[:text] = format(attachment[:text]) if attachment[:text] diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 57b533bad99..439ef0ce015 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,7 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_build_path(subject.project.namespace, + cancel_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index 3fec2c5d4db..b173c23fba4 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,7 +8,7 @@ module Gitlab end def details_path - namespace_project_build_path(subject.project.namespace, + namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c6139f1b716..e80f3263794 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 505f80848b2..56303e4cb17 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,7 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_build_path(subject.project.namespace, + retry_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 0b5199e5483..2778d6f3b52 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index fa462cbe095..c4c0623df6c 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -73,7 +73,7 @@ module Gitlab match = "" - stream.each_line do |line| + reverse_line do |line| matches = line.scan(regex) next unless matches.is_a?(Array) next if matches.empty? @@ -86,34 +86,39 @@ module Gitlab nil rescue # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now + # so we just silently ignore error for now end private - def read_last_lines(last_lines) - chunks = [] - pos = lines = 0 - max = stream.size - - # We want an extra line to make sure fist line has full contents - while lines <= last_lines && pos < max - pos += BUFFER_SIZE - - buf = - if pos <= max - stream.seek(-pos, IO::SEEK_END) - stream.read(BUFFER_SIZE) - else # Reached the head, read only left - stream.seek(0) - stream.read(BUFFER_SIZE - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) + def read_last_lines(limit) + to_enum(:reverse_line).first(limit).reverse.join + end + + def reverse_line + stream.seek(0, IO::SEEK_END) + debris = '' + + until (buf = read_backward(BUFFER_SIZE)).empty? + buf += debris + debris, *lines = buf.each_line.to_a + lines.reverse_each do |line| + yield(line.force_encoding('UTF-8')) + end end - chunks.join.lines.last(last_lines).join + yield(debris.force_encoding('UTF-8')) unless debris.empty? + end + + def read_backward(length) + cur_offset = stream.tell + start = cur_offset - length + start = 0 if start < 0 + + stream.seek(start, IO::SEEK_SET) + stream.read(cur_offset - start).tap do + stream.seek(start, IO::SEEK_SET) + end end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 82576d197fe..9e14b35b0f8 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -19,7 +19,7 @@ module Gitlab settings = ::ApplicationSetting.last end - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? + settings ||= ::ApplicationSetting.create_from_defaults end settings || in_memory_application_settings @@ -46,7 +46,8 @@ module Gitlab active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') + ActiveRecord::Base.connection.table_exists?('application_settings') && + !ActiveRecord::Migrator.needs_migration? rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e76c9abbe04..a412bb6dbd2 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -42,7 +42,7 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? + if supports_drop_index_concurrently? options = options.merge({ algorithm: :concurrently }) disable_statement_timeout end @@ -50,6 +50,39 @@ module Gitlab remove_index(table_name, options.merge({ column: column_name })) end + # Removes an existing index, concurrently when supported + # + # On PostgreSQL this method removes an index concurrently. + # + # Example: + # + # remove_concurrent_index :users, "index_X_by_Y" + # + # See Rails' `remove_index` for more info on the available arguments. + def remove_concurrent_index_by_name(table_name, index_name, options = {}) + if transaction_open? + raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if supports_drop_index_concurrently? + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout + end + + remove_index(table_name, options.merge({ name: index_name })) + end + + # Only available on Postgresql >= 9.2 + def supports_drop_index_concurrently? + return false unless Database.postgresql? + + version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + + version >= 90200 + end + # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking when using PostgreSQL. When diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb index c45ae8feb2c..3192bf6f667 100644 --- a/lib/gitlab/dependency_linker.rb +++ b/lib/gitlab/dependency_linker.rb @@ -1,7 +1,16 @@ module Gitlab module DependencyLinker LINKERS = [ - GemfileLinker + GemfileLinker, + GemspecLinker, + PackageJsonLinker, + ComposerJsonLinker, + PodfileLinker, + PodspecLinker, + PodspecJsonLinker, + CartfileLinker, + GodepsJsonLinker, + RequirementsTxtLinker ].freeze def self.linker(blob_name) diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 5f4027e7e81..7bbd154eb03 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -1,6 +1,9 @@ module Gitlab module DependencyLinker class BaseLinker + URL_REGEX = %r{https?://[^'" ]+}.freeze + REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze + class_attribute :file_type def self.support?(blob_name) @@ -26,59 +29,20 @@ module Gitlab private - def package_url(name) - raise NotImplementedError - end - def link_dependencies raise NotImplementedError end - def package_link(name, url = package_url(name)) - return name unless url - - %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + def license_url(name) + Licensee::License.find(name)&.url end - # Links package names in a method call or assignment string argument. - # - # Example: - # link_method_call("gem") - # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` - # - # link_method_call("gem", "specific_package") - # # Will link `specific_package` in `gem "specific_package"` - # - # link_method_call("github", /[^\/]+\/[^\/]+/) - # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` - # - # link_method_call(%w[add_dependency add_development_dependency]) - # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` - # - # link_method_call("name") - # # Will link `package` in `self.name = "package"` - def link_method_call(method_names, value = nil, &url_proc) - value = - case value - when String - Regexp.escape(value) - when nil - /[^'"]+/ - else - value - end - - method_names = Array(method_names).map { |name| Regexp.escape(name) } - - regex = %r{ - #{Regexp.union(method_names)} # Method name - \s* # Whitespace - [(=]? # Opening brace or equals sign - \s* # Whitespace - ['"](?<name>#{value})['"] # Package name in quotes - }x + def github_url(name) + "https://github.com/#{name}" + end - link_regex(regex, &url_proc) + def link_tag(name, url) + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} end # Links package names based on regex. @@ -86,13 +50,13 @@ module Gitlab # Example: # link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/) # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` - def link_regex(regex) + def link_regex(regex, &url_proc) highlighted_lines.map!.with_index do |rich_line, i| marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) marker.mark(regex, group: :name) do |text, left:, right:| - url = block_given? ? yield(text) : package_url(text) - package_link(text, url) + url = yield(text) + url ? link_tag(text, url) : text end end end @@ -104,6 +68,19 @@ module Gitlab def highlighted_lines @highlighted_lines ||= highlighted_text.lines end + + def regexp_for_value(value, default: /[^'" ]+/) + case value + when Array + Regexp.union(value.map { |v| regexp_for_value(v, default: default) }) + when String + Regexp.escape(value) + when Regexp + value + else + default + end + end end end end diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb new file mode 100644 index 00000000000..4f69f2c4ab2 --- /dev/null +++ b/lib/gitlab/dependency_linker/cartfile_linker.rb @@ -0,0 +1,14 @@ +module Gitlab + module DependencyLinker + class CartfileLinker < MethodLinker + self.file_type = :cartfile + + private + + def link_dependencies + link_method_call('github', REPO_REGEX, &method(:github_url)) + link_method_call(%w[github git binary], URL_REGEX, &:itself) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb new file mode 100644 index 00000000000..2fbde7da1b4 --- /dev/null +++ b/lib/gitlab/dependency_linker/cocoapods.rb @@ -0,0 +1,10 @@ +module Gitlab + module DependencyLinker + module Cocoapods + def package_url(name) + package = name.split("/", 2).first + "https://cocoapods.org/pods/#{package}" + end + end + end +end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb new file mode 100644 index 00000000000..0245bf4077a --- /dev/null +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + class ComposerJsonLinker < PackageJsonLinker + self.file_type = :composer_json + + private + + def link_packages + link_packages_at_key("require", &method(:package_url)) + link_packages_at_key("require-dev", &method(:package_url)) + end + + def package_url(name) + "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z} + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index 9b82e126528..d034ea67387 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -1,28 +1,31 @@ module Gitlab module DependencyLinker - class GemfileLinker < BaseLinker + class GemfileLinker < MethodLinker self.file_type = :gemfile private def link_dependencies - # Link `gem "package_name"` to https://rubygems.org/gems/package_name - link_method_call("gem") + link_urls + link_packages + end + def link_urls # Link `github: "user/repo"` to https://github.com/user/repo - link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name| - "https://github.com/#{name}" - end + link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url } + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) # Link `source "https://rubygems.org"` to https://rubygems.org - link_method_call("source", %r{https?://[^'"]+}) { |url| url } + link_method_call('source', URL_REGEX, &:itself) end - def package_url(name) - "https://rubygems.org/gems/#{name}" + def link_packages + # Link `gem "package_name"` to https://rubygems.org/gems/package_name + link_method_call('gem') do |name| + "https://rubygems.org/gems/#{name}" + end end end end diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb new file mode 100644 index 00000000000..f1783ee2ab4 --- /dev/null +++ b/lib/gitlab/dependency_linker/gemspec_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + class GemspecLinker < MethodLinker + self.file_type = :gemspec + + private + + def link_dependencies + link_method_call('homepage', URL_REGEX, &:itself) + link_method_call('license', &method(:license_url)) + + link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name| + "https://rubygems.org/gems/#{name}" + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb new file mode 100644 index 00000000000..fe091baee6d --- /dev/null +++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb @@ -0,0 +1,26 @@ +module Gitlab + module DependencyLinker + class GodepsJsonLinker < JsonLinker + NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze + + self.file_type = :godeps_json + + private + + def link_dependencies + link_json('ImportPath') do |path| + case path + when %r{\A(?<repo>gitlab\.com/#{NESTED_REPO_REGEX})\.git/(?<path>.+)\z}, + %r{\A(?<repo>git(lab|hub)\.com/#{REPO_REGEX})/(?<path>.+)\z} + + "https://#{$~[:repo]}/tree/master/#{$~[:path]}" + when /\Agolang\.org/ + "https://godoc.org/#{path}" + else + "https://#{path}" + end + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb new file mode 100644 index 00000000000..a8ef25233d8 --- /dev/null +++ b/lib/gitlab/dependency_linker/json_linker.rb @@ -0,0 +1,44 @@ +module Gitlab + module DependencyLinker + class JsonLinker < BaseLinker + def link + return highlighted_text unless json + + super + end + + private + + # Links package names in a JSON key or values. + # + # Example: + # link_json('name') + # # Will link `package` in `"name": "package"` + # + # link_json('name', 'specific_package') + # # Will link `specific_package` in `"name": "specific_package"` + # + # link_json('name', /[^\/]+\/[^\/]+/) + # # Will link `user/repo` in `"name": "user/repo"`, but not `"name": "package"` + # + # link_json('specific_package', '1.0.1', link: :key) + # # Will link `specific_package` in `"specific_package": "1.0.1"` + def link_json(key, value = nil, link: :value, &url_proc) + key = regexp_for_value(key, default: /[^" ]+/) + value = regexp_for_value(value, default: /[^" ]+/) + + if link == :value + value = /(?<name>#{value})/ + else + key = /(?<name>#{key})/ + end + + link_regex(/"#{key}":\s*"#{value}"/, &url_proc) + end + + def json + @json ||= JSON.parse(plain_text) rescue nil + end + end + end +end diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb new file mode 100644 index 00000000000..0ffa2a83c93 --- /dev/null +++ b/lib/gitlab/dependency_linker/method_linker.rb @@ -0,0 +1,39 @@ +module Gitlab + module DependencyLinker + class MethodLinker < BaseLinker + private + + # Links package names in a method call or assignment string argument. + # + # Example: + # link_method_call('gem') + # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` + # + # link_method_call('gem', 'specific_package') + # # Will link `specific_package` in `gem "specific_package"` + # + # link_method_call('github', /[^\/"]+\/[^\/"]+/) + # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` + # + # link_method_call(%w[add_dependency add_development_dependency]) + # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` + # + # link_method_call('name') + # # Will link `package` in `self.name = "package"` + def link_method_call(method_name, value = nil, &url_proc) + method_name = regexp_for_value(method_name) + value = regexp_for_value(value) + + regex = %r{ + #{method_name} # Method name + \s* # Whitespace + [(=]? # Opening brace or equals sign + \s* # Whitespace + ['"](?<name>#{value})['"] # Package name in quotes + }x + + link_regex(regex, &url_proc) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb new file mode 100644 index 00000000000..330c95f0880 --- /dev/null +++ b/lib/gitlab/dependency_linker/package_json_linker.rb @@ -0,0 +1,44 @@ +module Gitlab + module DependencyLinker + class PackageJsonLinker < JsonLinker + self.file_type = :package_json + + private + + def link_dependencies + link_json('name', json["name"], &method(:package_url)) + link_json('license', &method(:license_url)) + link_json(%w[homepage url], URL_REGEX, &:itself) + + link_packages + end + + def link_packages + link_packages_at_key("dependencies", &method(:package_url)) + link_packages_at_key("devDependencies", &method(:package_url)) + end + + def link_packages_at_key(key, &url_proc) + dependencies = json[key] + return unless dependencies + + dependencies.each do |name, version| + link_json(name, version, link: :key, &url_proc) + + link_json(name) do |value| + case value + when /\A#{URL_REGEX}\z/ + value + when /\A#{REPO_REGEX}\z/ + github_url(value) + end + end + end + end + + def package_url(name) + "https://npmjs.com/package/#{name}" + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb new file mode 100644 index 00000000000..60ad166ea17 --- /dev/null +++ b/lib/gitlab/dependency_linker/podfile_linker.rb @@ -0,0 +1,15 @@ +module Gitlab + module DependencyLinker + class PodfileLinker < GemfileLinker + include Cocoapods + + self.file_type = :podfile + + private + + def link_packages + link_method_call('pod', &method(:package_url)) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb new file mode 100644 index 00000000000..d82237ed3f1 --- /dev/null +++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb @@ -0,0 +1,32 @@ +module Gitlab + module DependencyLinker + class PodspecJsonLinker < JsonLinker + include Cocoapods + + self.file_type = :podspec_json + + private + + def link_dependencies + link_json('name', json["name"], &method(:package_url)) + link_json('license', &method(:license_url)) + link_json(%w[homepage git], URL_REGEX, &:itself) + + link_packages_at_key("dependencies", &method(:package_url)) + + json["subspecs"]&.each do |subspec| + link_packages_at_key("dependencies", subspec, &method(:package_url)) + end + end + + def link_packages_at_key(key, root = json, &url_proc) + dependencies = root[key] + return unless dependencies + + dependencies.each do |name, _| + link_regex(/"(?<name>#{Regexp.escape(name)})":\s*\[/, &url_proc) + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb new file mode 100644 index 00000000000..a52c7a02439 --- /dev/null +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -0,0 +1,24 @@ +module Gitlab + module DependencyLinker + class PodspecLinker < MethodLinker + include Cocoapods + + STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze + + self.file_type = :podspec + + private + + def link_dependencies + link_method_call('homepage', URL_REGEX, &:itself) + + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + + link_method_call('license', &method(:license_url)) + link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) + + link_method_call(%w[name dependency], &method(:package_url)) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb new file mode 100644 index 00000000000..2e197e5cd94 --- /dev/null +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -0,0 +1,17 @@ +module Gitlab + module DependencyLinker + class RequirementsTxtLinker < BaseLinker + self.file_type = :requirements_txt + + private + + def link_dependencies + link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=;\[]+)/) do |name| + "https://pypi.python.org/pypi/#{name}" + end + + link_regex(%r{^(?<name>https?://[^ ]+)}, &:itself) + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index c6bf25b5874..2aef7fdaa35 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,16 +1,17 @@ module Gitlab module Diff class File - attr_reader :diff, :repository, :diff_refs + attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs - delegate :new_file, :deleted_file, :renamed_file, - :old_path, :new_path, :a_mode, :b_mode, + delegate :new_file?, :deleted_file?, :renamed_file?, + :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false - def initialize(diff, repository:, diff_refs: nil) + def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) @diff = diff @repository = repository @diff_refs = diff_refs + @fallback_diff_refs = fallback_diff_refs end def position(line) @@ -49,24 +50,60 @@ module Gitlab line_code(line) if line end + def old_sha + diff_refs&.base_sha + end + + def new_sha + diff_refs&.head_sha + end + + def content_sha + return old_content_sha if deleted_file? + return @content_sha if defined?(@content_sha) + + refs = diff_refs || fallback_diff_refs + @content_sha = refs&.head_sha + end + def content_commit - return unless diff_refs + return @content_commit if defined?(@content_commit) + + sha = content_sha + @content_commit = repository.commit(sha) if sha + end + + def old_content_sha + return if new_file? + return @old_content_sha if defined?(@old_content_sha) - repository.commit(deleted_file ? old_ref : new_ref) + refs = diff_refs || fallback_diff_refs + @old_content_sha = refs&.base_sha end def old_content_commit - return unless diff_refs + return @old_content_commit if defined?(@old_content_commit) - repository.commit(old_ref) + sha = old_content_sha + @old_content_commit = repository.commit(sha) if sha end - def old_ref - diff_refs.try(:base_sha) + def blob + return @blob if defined?(@blob) + + sha = content_sha + return @blob = nil unless sha + + repository.blob_at(sha, file_path) end - def new_ref - diff_refs.try(:head_sha) + def old_blob + return @old_blob if defined?(@old_blob) + + sha = old_content_sha + return @old_blob = nil unless sha + + @old_blob = repository.blob_at(sha, old_path) end attr_writer :highlighted_diff_lines @@ -85,10 +122,6 @@ module Gitlab @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end - def mode_changed? - a_mode && b_mode && a_mode != b_mode - end - def raw_diff diff.diff.to_s end @@ -117,20 +150,8 @@ module Gitlab diff_lines.count(&:removed?) end - def old_blob(commit = old_content_commit) - return unless commit - - repository.blob_at(commit.id, old_path) - end - - def blob(commit = content_commit) - return unless commit - - repository.blob_at(commit.id, file_path) - end - def file_identifier - "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" + "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}" end end end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 2b9fc65b985..a6007ebf531 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -2,32 +2,41 @@ module Gitlab module Diff module FileCollection class Base - attr_reader :project, :diff_options, :diff_view, :diff_refs + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) end - def initialize(diffable, project:, diff_options: nil, diff_refs: nil) + def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff_options = self.class.default_options.merge(diff_options || {}) - @diffable = diffable - @diffs = diffable.raw_diffs(diff_options) - @project = project + @diffable = diffable + @diffs = diffable.raw_diffs(diff_options) + @project = project @diff_options = diff_options - @diff_refs = diff_refs + @diff_refs = diff_refs + @fallback_diff_refs = fallback_diff_refs end def diff_files @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } end + def diff_file_with_old_path(old_path) + diff_files.find { |diff_file| diff_file.old_path == old_path } + end + + def diff_file_with_new_path(new_path) + diff_files.find { |diff_file| diff_file.new_path == new_path } + end + private def decorate_diff!(diff) - Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs) + Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 0bd226ef050..9a58b500a2c 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -8,7 +8,8 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, diff_options: diff_options, - diff_refs: merge_request_diff.diff_refs) + diff_refs: merge_request_diff.diff_refs, + fallback_diff_refs: merge_request_diff.fallback_diff_refs) end def diff_files diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 7db896522a9..ed2f541977a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,7 +3,7 @@ module Gitlab class Highlight attr_reader :diff_file, :diff_lines, :raw_lines, :repository - delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff + delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff def initialize(diff_lines, repository: nil) @repository = repository @@ -61,12 +61,12 @@ module Gitlab def old_lines return unless diff_file - @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path) + @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_sha, diff_old_path) end def new_lines return unless diff_file - @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path) + @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_sha, diff_new_path) end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0a15c6d9358..bd52ae47e9f 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -59,6 +59,10 @@ module Gitlab type == 'match' end + def discussable? + !['match', 'new-nonewline', 'old-nonewline'].include?(type) + end + def as_json(opts = nil) { type: type, diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index fc728123c97..4d96778a2b2 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -12,20 +12,26 @@ module Gitlab attr_reader :head_sha def initialize(attrs = {}) + if diff_file = attrs[:diff_file] + attrs[:diff_refs] = diff_file.diff_refs + attrs[:old_path] = diff_file.old_path + attrs[:new_path] = diff_file.new_path + end + + if diff_refs = attrs[:diff_refs] + attrs[:base_sha] = diff_refs.base_sha + attrs[:start_sha] = diff_refs.start_sha + attrs[:head_sha] = diff_refs.head_sha + end + @old_path = attrs[:old_path] @new_path = attrs[:new_path] + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + @old_line = attrs[:old_line] @new_line = attrs[:new_line] - - if attrs[:diff_refs] - @base_sha = attrs[:diff_refs].base_sha - @start_sha = attrs[:diff_refs].start_sha - @head_sha = attrs[:diff_refs].head_sha - else - @base_sha = attrs[:base_sha] - @start_sha = attrs[:start_sha] - @head_sha = attrs[:head_sha] - end end # `Gitlab::Diff::Position` objects are stored as serialized attributes in @@ -129,11 +135,11 @@ module Gitlab end def diff_line(repository) - @diff_line ||= diff_file(repository).line_for_position(self) + @diff_line ||= diff_file(repository)&.line_for_position(self) end def line_code(repository) - @line_code ||= diff_file(repository).line_code_for_position(self) + @line_code ||= diff_file(repository)&.line_code_for_position(self) end private diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index e89ff238ec7..dcabb5f7fe5 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -3,21 +3,21 @@ module Gitlab module Diff class PositionTracer - attr_accessor :repository + attr_accessor :project attr_accessor :old_diff_refs attr_accessor :new_diff_refs attr_accessor :paths - def initialize(repository:, old_diff_refs:, new_diff_refs:, paths: nil) - @repository = repository + def initialize(project:, old_diff_refs:, new_diff_refs:, paths: nil) + @project = project @old_diff_refs = old_diff_refs @new_diff_refs = new_diff_refs @paths = paths end - def trace(old_position) + def trace(ab_position) return unless old_diff_refs&.complete? && new_diff_refs&.complete? - return unless old_position.diff_refs == old_diff_refs + return unless ab_position.diff_refs == old_diff_refs # Suppose we have an MR with source branch `feature` and target branch `master`. # When the MR was created, the head of `master` was commit A, and the @@ -44,14 +44,16 @@ module Gitlab # # For diff notes for diff A->B, the position looks like this: # Position - # base_sha - ID of commit A + # start_sha - ID of commit A # head_sha - ID of commit B + # base_sha - ID of base commit of A and B # old_path - path as of A (nil if file was newly created) # new_path - path as of B (nil if file was deleted) # old_line - line number as of A (nil if file was newly created) # new_line - line number as of B (nil if file was deleted) # - # We can easily update `base_sha` and `head_sha` to hold the IDs of commits C and D, + # We can easily update `start_sha` and `head_sha` to hold the IDs of + # commits C and D, and can trivially determine `base_sha` based on those, # but need to find the paths and line numbers as of C and D. # # If the file was unchanged or newly created in A->B, the path as of D can be found @@ -68,107 +70,161 @@ module Gitlab # by generating diff A->C ("base to base"), finding the diff file with # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. # The path as of D can be found by taking diff C->D, finding the diff file - # with that same `old_path` and taking `diff_file.new_path`. + # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`. # The line number as of C can be found by using the LineMapper on diff A->C # and providing the line number as of A. # The line number as of D can be found by using the LineMapper on diff C->D # and providing the line number as of C. - results = nil - results ||= trace_added_line(old_position) if old_position.added? || old_position.unchanged? - results ||= trace_removed_line(old_position) if old_position.removed? || old_position.unchanged? - - return unless results - - file_diff, old_line, new_line = results - - new_position = Position.new( - old_path: file_diff.old_path, - new_path: file_diff.new_path, - head_sha: new_diff_refs.head_sha, - start_sha: new_diff_refs.start_sha, - base_sha: new_diff_refs.base_sha, - old_line: old_line, - new_line: new_line - ) - - # If a position is found, but is not actually contained in the diff, for example - # because it was an unchanged line in the context of a change that was undone, - # we cannot return this as a successful trace. - return unless new_position.diff_line(repository) - - new_position + if ab_position.added? + trace_added_line(ab_position) + elsif ab_position.removed? + trace_removed_line(ab_position) + else # unchanged + trace_unchanged_line(ab_position) + end end private - def trace_added_line(old_position) - file_path = old_position.new_path - - return unless diff_head_to_head - - file_head_to_head = diff_head_to_head.find { |diff_file| diff_file.old_path == file_path } - - file_path = file_head_to_head.new_path if file_head_to_head - - new_line = LineMapper.new(file_head_to_head).old_to_new(old_position.new_line) - - return unless new_line - - file_diff = new_diffs.find { |diff_file| diff_file.new_path == file_path } - return unless file_diff - - old_line = LineMapper.new(file_diff).new_to_old(new_line) - - [file_diff, old_line, new_line] + def trace_added_line(ab_position) + b_path = ab_position.new_path + b_line = ab_position.new_line + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_path = bd_diff&.new_path || b_path + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + if d_line + cd_diff = cd_diffs.diff_file_with_new_path(d_path) + + c_path = cd_diff&.old_path || d_path + c_line = LineMapper.new(cd_diff).new_to_old(d_line) + + if c_line + # If the line is still in D but also in C, it has turned from an + # added line into an unchanged one. + new_position = position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff, so we treat it as outdated. + ac_diff = ac_diffs.diff_file_with_new_path(c_path) + + { position: position(ac_diff, nil, c_line), outdated: true } + end + else + # If the line is still in D and not in C, it is still added. + { position: position(cd_diff, nil, d_line), outdated: false } + end + else + # If the line is no longer in D, it has been removed from the MR. + { position: position(bd_diff, b_line, nil), outdated: true } + end end - def trace_removed_line(old_position) - file_path = old_position.old_path + def trace_removed_line(ab_position) + a_path = ab_position.old_path + a_line = ab_position.old_line - return unless diff_base_to_base + ac_diff = ac_diffs.diff_file_with_old_path(a_path) - file_base_to_base = diff_base_to_base.find { |diff_file| diff_file.old_path == file_path } + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) - file_path = file_base_to_base.old_path if file_base_to_base + if c_line + cd_diff = cd_diffs.diff_file_with_old_path(c_path) - old_line = LineMapper.new(file_base_to_base).old_to_new(old_position.old_line) + d_path = cd_diff&.new_path || c_path + d_line = LineMapper.new(cd_diff).old_to_new(c_line) - return unless old_line + if d_line + # If the line is still in C but also in D, it has turned from a + # removed line into an unchanged one. + bd_diff = bd_diffs.diff_file_with_new_path(d_path) - file_diff = new_diffs.find { |diff_file| diff_file.old_path == file_path } - return unless file_diff - - new_line = LineMapper.new(file_diff).old_to_new(old_line) + { position: position(bd_diff, nil, d_line), outdated: true } + else + # If the line is still in C and not in D, it is still removed. + { position: position(cd_diff, c_line, nil), outdated: false } + end + else + # If the line is no longer in C, it has been removed outside of the MR. + { position: position(ac_diff, a_line, nil), outdated: true } + end + end - [file_diff, old_line, new_line] + def trace_unchanged_line(ab_position) + a_path = ab_position.old_path + a_line = ab_position.old_line + b_path = ab_position.new_path + b_line = ab_position.new_line + + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + cd_diff = cd_diffs.diff_file_with_old_path(c_path) + + if c_line && d_line + # If the line is still in C and D, it is still unchanged. + new_position = position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff or any change on the BD diff, + # so we treat it as outdated. + { position: nil, outdated: true } + end + elsif d_line # && !c_line + # If the line is still in D but no longer in C, it has turned from + # an unchanged line into an added one. + # We don't treat this as outdated since the line is still in the MR. + { position: position(cd_diff, nil, d_line), outdated: false } + else # !d_line && (c_line || !c_line) + # If the line is no longer in D, it has turned from an unchanged line + # into a removed one. + { position: position(bd_diff, b_line, nil), outdated: true } + end end - def diff_base_to_base - @diff_base_to_base ||= diff_files(old_diff_refs.base_sha || old_diff_refs.start_sha, new_diff_refs.base_sha || new_diff_refs.start_sha) + def ac_diffs + @ac_diffs ||= compare( + old_diff_refs.base_sha || old_diff_refs.start_sha, + new_diff_refs.base_sha || new_diff_refs.start_sha, + straight: true + ) end - def diff_head_to_head - @diff_head_to_head ||= diff_files(old_diff_refs.head_sha, new_diff_refs.head_sha) + def bd_diffs + @bd_diffs ||= compare(old_diff_refs.head_sha, new_diff_refs.head_sha, straight: true) end - def new_diffs - @new_diffs ||= diff_files(new_diff_refs.start_sha, new_diff_refs.head_sha, use_base: true) + def cd_diffs + @cd_diffs ||= compare(new_diff_refs.start_sha, new_diff_refs.head_sha) end - def diff_files(start_sha, head_sha, use_base: false) - base_sha = self.repository.merge_base(start_sha, head_sha) || start_sha + def compare(start_sha, head_sha, straight: false) + compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + compare.diffs(paths: paths) + end - diffs = self.repository.raw_repository.diff( - use_base ? base_sha : start_sha, - head_sha, - {}, - *paths - ) + def position(diff_file, old_line, new_line) + Position.new(diff_file: diff_file, old_line: old_line, new_line: new_line) + end - diffs.decorate! do |diff| - Gitlab::Diff::File.new(diff, repository: self.repository) - end + def valid_position?(position) + !!position.diff_line(project.repository) end end end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 6c69cd9e6a9..ea035e33eff 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -42,7 +42,7 @@ module Gitlab return unless compare # This diff is more moderated in number of files and lines - @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files + @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, expanded: true).diff_files end def diffs_count diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb new file mode 100644 index 00000000000..dbe28e6bb93 --- /dev/null +++ b/lib/gitlab/encoding_helper.rb @@ -0,0 +1,62 @@ +module Gitlab + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end +end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index ba31041d0c1..15ae24f62c8 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -7,11 +7,32 @@ module Gitlab # - Don't contain a reserved word (expect for the words used in the # regex itself) # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route +<<<<<<< HEAD + # - Ending in `issues/id`/rendered_title` for the `issue_title` route +<<<<<<< HEAD +======= # - Ending in `issues/id`/realtime_changes` for the `issue_title` route +>>>>>>> 2f62af6... Restore original comment [ci skip] USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes +<<<<<<< HEAD +<<<<<<< HEAD commit pipelines merge_requests new].freeze +======= + commit pipelines merge_requests builds + new].freeze + +>>>>>>> 47a0276... Initial implementation for real time job view + RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) +======= +======= + USED_IN_ROUTES = %w[noteable issue notes issues rendered_title +>>>>>>> 4535d52... Use etag caching for environments JSON + commit pipelines merge_requests new + environments].freeze RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) +>>>>>>> ebede2b... Use etag caching for environments JSON ROUTES = [ Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), @@ -38,8 +59,21 @@ module Gitlab 'project_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z), 'project_pipeline' + ), + Gitlab::EtagCaching::Router::Route.new( +<<<<<<< HEAD +<<<<<<< HEAD + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), +======= + %r(^(?!.*(#{RESERVED_WORDS})).*/environments\.json\z), +>>>>>>> 4535d52... Use etag caching for environments JSON + 'environments' +======= + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z), + 'project_build' +>>>>>>> 47a0276... Initial implementation for real time job view ) ].freeze diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 58193391926..66829a03c2e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Blame - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_reader :lines, :blames diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index c1b31618e0d..d60e607b02b 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Blob include Linguist::BlobHelper - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection @@ -88,6 +88,7 @@ module Gitlab new( id: blob_entry[:oid], name: blob_entry[:name], + size: 0, data: '', path: path, commit_id: sha diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 297531db4cc..bb04731f08c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Commit - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :raw_commit, :head, :refs diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 31d1b66b4f7..8926aa19925 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -3,7 +3,7 @@ module Gitlab module Git class Diff TimeoutError = Class.new(StandardError) - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Diff properties attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff @@ -11,15 +11,34 @@ module Gitlab # Stats properties attr_accessor :new_file, :renamed_file, :deleted_file - attr_accessor :too_large + alias_method :new_file?, :new_file + alias_method :deleted_file?, :deleted_file + alias_method :renamed_file?, :renamed_file - # The maximum size of a diff to display. - DIFF_SIZE_LIMIT = 102400 # 100 KB + attr_accessor :expanded - # The maximum size before a diff is collapsed. - DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + # We need this accessor because of `to_hash` and `init_from_hash` + attr_accessor :too_large class << self + # The maximum size of a diff to display. + def size_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 200.kilobytes + else + 100.kilobytes + end + end + + # The maximum size before a diff is collapsed. + def collapse_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 100.kilobytes + else + 10.kilobytes + end + end + def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -148,7 +167,7 @@ module Gitlab :include_untracked_content, :skip_binary_check, :include_typechange, :include_typechange_trees, :ignore_filemode, :recurse_ignored_dirs, :paths, - :max_files, :max_lines, :all_diffs, :no_collapse] + :max_files, :max_lines, :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -173,16 +192,18 @@ module Gitlab end end - def initialize(raw_diff, collapse: false) + def initialize(raw_diff, expanded: true) + @expanded = expanded + case raw_diff when Hash init_from_hash(raw_diff) - prune_diff_if_eligible(collapse) + prune_diff_if_eligible when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff, collapse: collapse) - when Gitaly::CommitDiffResponse + init_from_rugged(raw_diff) + when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) - prune_diff_if_eligible(collapse) + prune_diff_if_eligible when Gitaly::CommitDelta init_from_gitaly(raw_diff) when nil @@ -208,6 +229,10 @@ module Gitlab hash end + def mode_changed? + a_mode && b_mode && a_mode != b_mode + end + def submodule? a_mode == '160000' || b_mode == '160000' end @@ -218,17 +243,13 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.size_limit else @too_large end end - def collapsible? - @diff.bytesize >= DIFF_COLLAPSE_LIMIT - end - - def prune_large_diff! + def too_large! @diff = '' @line_count = 0 @too_large = true @@ -236,10 +257,11 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - false + + @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit end - def prune_collapsed_diff! + def collapse! @diff = '' @line_count = 0 @collapsed = true @@ -247,9 +269,9 @@ module Gitlab private - def init_from_rugged(rugged, collapse: false) + def init_from_rugged(rugged) if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged, collapse: collapse) + init_from_rugged_patch(rugged) d = rugged.delta else d = rugged @@ -264,10 +286,10 @@ module Gitlab @deleted_file = d.deleted? end - def init_from_rugged_patch(patch, collapse: false) + def init_from_rugged_patch(patch) # Don't bother initializing diffs that are too large. If a diff is # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch, collapse) + return if !patch.delta.binary? && prune_large_patch(patch) @diff = encode!(strip_diff_headers(patch.to_s)) end @@ -280,40 +302,43 @@ module Gitlab end end - def init_from_gitaly(msg) - @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) - @new_path = encode!(msg.to_path.dup) - @old_path = encode!(msg.from_path.dup) - @a_mode = msg.old_mode.to_s(8) - @b_mode = msg.new_mode.to_s(8) - @new_file = msg.from_id == BLANK_SHA - @renamed_file = msg.from_path != msg.to_path - @deleted_file = msg.to_id == BLANK_SHA + def init_from_gitaly(diff) + @diff = diff.patch if diff.respond_to?(:patch) + @new_path = encode!(diff.to_path.dup) + @old_path = encode!(diff.from_path.dup) + @a_mode = diff.old_mode.to_s(8) + @b_mode = diff.new_mode.to_s(8) + @new_file = diff.from_id == BLANK_SHA + @renamed_file = diff.from_path != diff.to_path + @deleted_file = diff.to_id == BLANK_SHA end - def prune_diff_if_eligible(collapse = false) - prune_large_diff! if too_large? - prune_collapsed_diff! if collapse && collapsible? + def prune_diff_if_eligible + if too_large? + too_large! + elsif collapsed? + collapse! + end end # If the patch surpasses any of the diff limits it calls the appropiate # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch, collapse) + def prune_large_patch(patch) size = 0 patch.each_hunk do |hunk| hunk.each_line do |line| size += line.content.bytesize - if size >= DIFF_SIZE_LIMIT - prune_large_diff! + if size >= self.class.size_limit + too_large! return true end end end - if collapse && size >= DIFF_COLLAPSE_LIMIT - prune_collapsed_diff! + if !expanded && size >= self.class.collapse_limit + collapse! return true end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index bcbad8ec829..334e06a6eca 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -9,32 +9,29 @@ module Gitlab @iterator = iterator @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) - @max_bytes = @max_files * 5120 # Average 5 KB per file + @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min - @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file - @all_diffs = !!options.fetch(:all_diffs, false) - @no_collapse = !!options.fetch(:no_collapse, true) + @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file + @enforce_limits = !!options.fetch(:limits, true) + @expanded = !!options.fetch(:expanded, true) @line_count = 0 @byte_count = 0 @overflow = false + @empty = true @array = Array.new end def each(&block) - if @populated - # @iterator.each is slower than just iterating the array in place - @array.each(&block) - else - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do - each_patch(&block) - end + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do + each_patch(&block) end end def empty? - !@iterator.any? + any? # Make sure the iterator has been exercised + @empty end def overflow? @@ -60,17 +57,17 @@ module Gitlab collection = each_with_index do |element, i| @array[i] = yield(element) end - @populated = true collection end + alias_method :to_ary, :to_a + private def populate! return if @populated each { nil } # force a loop through all diffs - @populated = true nil end @@ -79,33 +76,35 @@ module Gitlab end def each_patch - @iterator.each_with_index do |raw, i| - # First yield cached Diff instances from @array - if @array[i] - yield @array[i] - next - end + i = 0 + @array.each do |diff| + yield diff + i += 1 + end + + return if @overflow + return if @iterator.nil? - # We have exhausted @array, time to create new Diff instances or stop. - break if @overflow + @iterator.each do |raw| + @empty = false - if !@all_diffs && i >= @max_files + if @enforce_limits && i >= @max_files @overflow = true break end - collapse = !@all_diffs && !@no_collapse + expanded = !@enforce_limits || @expanded - diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + diff = Gitlab::Git::Diff.new(raw, expanded: expanded) - if collapse && over_safe_limits?(i) - diff.prune_collapsed_diff! + if !expanded && over_safe_limits?(i) + diff.collapse! end @line_count += diff.line_count @byte_count += diff.diff.bytesize - if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true @@ -113,7 +112,13 @@ module Gitlab end yield @array[i] = diff + i += 1 end + + @populated = true + + # Allow iterator to be garbage-collected. It cannot be reused anyway. + @iterator = nil end end end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb deleted file mode 100644 index f918074cb14..00000000000 --- a/lib/gitlab/git/encoding_helper.rb +++ /dev/null @@ -1,64 +0,0 @@ -module Gitlab - module Git - module EncodingHelper - extend self - - # This threshold is carefully tweaked to prevent usage of encodings detected - # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, - # we're better off sticking with utf8 encoding. - # Reason: git diff can return strings with invalid utf8 byte sequences if it - # truncates a diff in the middle of a multibyte character. In this case - # CharlockHolmes will try to guess the encoding and will likely suggest an - # obscure encoding with low confidence. - # There is a lot more info with this merge request: - # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 - - def encode!(message) - return nil unless message.respond_to? :force_encoding - - # if message is utf-8 encoding, just return it - message.force_encoding("UTF-8") - return message if message.valid_encoding? - - # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary - - # force detected encoding if we have sufficient confidence. - if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD - message.force_encoding(detect[:encoding]) - end - - # encode and clean the bad chars - message.replace clean(message) - rescue - encoding = detect ? detect[:encoding] : "unknown" - "--broken encoding: #{encoding}" - end - - def encode_utf8(message) - detect = CharlockHolmes::EncodingDetector.detect(message) - if detect - begin - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') - rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") - - '' - end - else - clean(message) - end - end - - private - - def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") - .encode("UTF-8") - .gsub("\0".encode("UTF-8"), "") - end - end - end -end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 37ef6836742..ebf7393dc61 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Ref - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index b9f1ac144b6..9d6adbdb4ac 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1006,31 +1006,39 @@ module Gitlab # Parses the contents of a .gitmodules file and returns a hash of # submodule information. def parse_gitmodules(commit, content) - results = {} + modules = {} - current = "" - content.split("\n").each do |txt| - if txt =~ /^\s*\[/ - current = txt.match(/(?<=").*(?=")/)[0] - results[current] = {} - else - next unless results[current] - match_data = txt.match(/(\w+)\s*=\s*(.*)/) - next unless match_data - target = match_data[2].chomp - results[current][match_data[1]] = target + name = nil + content.each_line do |line| + case line.strip + when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header + name = $~[:name] + modules[name] = {} + when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair + key = $~[:key] + value = $~[:value].chomp + + next unless name && modules[name] + + modules[name][key] = value - if match_data[1] == "path" + if key == 'path' begin - results[current]["id"] = blob_content(commit, target) + modules[name]['id'] = blob_content(commit, value) rescue InvalidBlobName - results.delete(current) + # The current entry is invalid + modules.delete(name) + name = nil end end + when /\A#/ # Comment + next + else # Invalid line + name = nil end end - results + modules end # Returns true if +commit+ introduced changes to +path+, using commit @@ -1086,7 +1094,12 @@ module Gitlab elsif tmp_entry.nil? return nil else - tmp_entry = rugged.lookup(tmp_entry[:oid]) + begin + tmp_entry = rugged.lookup(tmp_entry[:oid]) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + return nil + end + return nil unless tmp_entry.type == :tree tmp_entry = tmp_entry[dir] end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index d41256d9a84..b9afa05c819 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Tree - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :id, :root_id, :name, :path, :type, :mode, :commit_id, :submodule_url diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 72466700c05..2343446bf22 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -2,6 +2,12 @@ require 'gitaly' module Gitlab module GitalyClient + module MigrationStatus + DISABLED = 1 + OPT_IN = 2 + OPT_OUT = 3 + end + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze MUTEX = Mutex.new @@ -46,8 +52,20 @@ module Gitlab Gitlab.config.gitaly.enabled end - def self.feature_enabled?(feature) - enabled? && ENV["GITALY_#{feature.upcase}"] == '1' + def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) + return false if !enabled? || status == MigrationStatus::DISABLED + + feature = Feature.get("gitaly_#{feature}") + + # If the feature hasn't been set, turn it on if it's opt-out + return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature) + + if feature.percentage_of_time_value > 0 + # Probabilistically enable this feature + return Random.rand() * 100 < feature.percentage_of_time_value + end + + feature.enabled? end def self.migrate(feature) diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 4491903d788..ba3da781dad 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -26,7 +26,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) - Gitlab::Git::DiffCollection.new(response, options) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) end def commit_deltas(commit) diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb new file mode 100644 index 00000000000..1e117b7e74a --- /dev/null +++ b/lib/gitlab/gitaly_client/diff.rb @@ -0,0 +1,21 @@ +module Gitlab + module GitalyClient + class Diff + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + + attr_accessor(*FIELDS) + + def initialize(params) + params.each do |key, val| + public_send(:"#{key}=", val) + end + end + + def ==(other) + FIELDS.all? do |field| + public_send(field) == other.public_send(field) + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb new file mode 100644 index 00000000000..d84e8d752dc --- /dev/null +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + class DiffStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_diff = nil + + @rpc_response.each do |diff_msg| + if current_diff.nil? + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params[:patch] = diff_msg.raw_patch_data + + current_diff = GitalyClient::Diff.new(diff_params) + else + current_diff.patch += diff_msg.raw_patch_data + end + + if diff_msg.end_of_patch + yield current_diff + current_diff = nil + end + end + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 6200bd460ea..319633656ff 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -1,3 +1,5 @@ +# rubocop:disable Metrics/AbcSize + module Gitlab module GonHelper def add_gon_variables @@ -12,6 +14,8 @@ module Gitlab gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url + gon.revision = Gitlab::REVISION + gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb index 890bd9a3554..b1dbf554e41 100644 --- a/lib/gitlab/google_code_import/client.rb +++ b/lib/gitlab/google_code_import/client.rb @@ -14,7 +14,7 @@ module Gitlab end def valid? - raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects") + raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.key?("projects") end def repos diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 1b43440673c..ab38c0c3e34 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -95,7 +95,7 @@ module Gitlab labels = import_issue_labels(raw_issue) assignee_id = nil - if raw_issue.has_key?("owner") + if raw_issue.key?("owner") username = user_map[raw_issue["owner"]["name"]] if username.start_with?("@") @@ -144,7 +144,7 @@ module Gitlab def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift - next if raw_comment.has_key?("deletedBy") + next if raw_comment.key?("deletedBy") content = format_content(raw_comment["content"]) updates = format_updates(raw_comment["updates"]) @@ -235,15 +235,15 @@ module Gitlab def format_updates(raw_updates) updates = [] - if raw_updates.has_key?("status") + if raw_updates.key?("status") updates << "*Status: #{raw_updates["status"]}*" end - if raw_updates.has_key?("owner") + if raw_updates.key?("owner") updates << "*Owner: #{user_map[raw_updates["owner"]]}*" end - if raw_updates.has_key?("cc") + if raw_updates.key?("cc") cc = raw_updates["cc"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -255,7 +255,7 @@ module Gitlab updates << "*Cc: #{cc.join(", ")}*" end - if raw_updates.has_key?("labels") + if raw_updates.key?("labels") labels = raw_updates["labels"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -267,11 +267,11 @@ module Gitlab updates << "*Labels: #{labels.join(", ")}*" end - if raw_updates.has_key?("mergedInto") + if raw_updates.key?("mergedInto") updates << "*Merged into: ##{raw_updates["mergedInto"]}*" end - if raw_updates.has_key?("blockedOn") + if raw_updates.key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end @@ -279,7 +279,7 @@ module Gitlab updates << "*Blocked on: #{blocked_ons.join(", ")}*" end - if raw_updates.has_key?("blocking") + if raw_updates.key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb new file mode 100644 index 00000000000..e9d5d52cabb --- /dev/null +++ b/lib/gitlab/group_hierarchy.rb @@ -0,0 +1,104 @@ +module Gitlab + # Retrieving of parent or child groups based on a base ActiveRecord relation. + # + # This class uses recursive CTEs and as a result will only work on PostgreSQL. + class GroupHierarchy + attr_reader :base, :model + + # base - An instance of ActiveRecord::Relation for which to get parent or + # child groups. + def initialize(base) + @base = base + @model = base.model + end + + # Returns a relation that includes the base set of groups and all their + # ancestors (recursively). + def base_and_ancestors + return model.none unless Group.supports_nested_groups? + + base_and_ancestors_cte.apply_to(model.all) + end + + # Returns a relation that includes the base set of groups and all their + # descendants (recursively). + def base_and_descendants + return model.none unless Group.supports_nested_groups? + + base_and_descendants_cte.apply_to(model.all) + end + + # Returns a relation that includes the base groups, their ancestors, and the + # descendants of the base groups. + # + # The resulting query will roughly look like the following: + # + # WITH RECURSIVE ancestors AS ( ... ), + # descendants AS ( ... ) + # SELECT * + # FROM ( + # SELECT * + # FROM ancestors namespaces + # + # UNION + # + # SELECT * + # FROM descendants namespaces + # ) groups; + # + # Using this approach allows us to further add criteria to the relation with + # Rails thinking it's selecting data the usual way. + def all_groups + return base unless Group.supports_nested_groups? + + ancestors = base_and_ancestors_cte + descendants = base_and_descendants_cte + + ancestors_table = ancestors.alias_to(groups_table) + descendants_table = descendants.alias_to(groups_table) + + union = SQL::Union.new([model.unscoped.from(ancestors_table), + model.unscoped.from(descendants_table)]) + + model. + unscoped. + with. + recursive(ancestors.to_arel, descendants.to_arel). + from("(#{union.to_sql}) #{model.table_name}") + end + + private + + def base_and_ancestors_cte + cte = SQL::RecursiveCTE.new(:base_and_ancestors) + + cte << base.except(:order) + + # Recursively get all the ancestors of the base set. + cte << model. + from([groups_table, cte.table]). + where(groups_table[:id].eq(cte.table[:parent_id])). + except(:order) + + cte + end + + def base_and_descendants_cte + cte = SQL::RecursiveCTE.new(:base_and_descendants) + + cte << base.except(:order) + + # Recursively get all the descendants of the base set. + cte << model. + from([groups_table, cte.table]). + where(groups_table[:parent_id].eq(cte.table[:id])). + except(:order) + + cte + end + + def groups_table + model.arel_table + end + end +end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index df962d203b7..e78b7f22e03 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -2,6 +2,9 @@ module Gitlab module HealthChecks class FsShardsCheck extend BaseAbstractCheck + RANDOM_STRING = SecureRandom.hex(1000).freeze + COMMAND_TIMEOUT = '1'.freeze + TIMEOUT_EXECUTABLE = 'timeout'.freeze class << self def readiness @@ -41,8 +44,6 @@ module Gitlab private - RANDOM_STRING = SecureRandom.hex(1000).freeze - def operation_metrics(ok_metric, latency_metric, operation, **labels) with_timing operation do |result, elapsed| [ @@ -63,8 +64,8 @@ module Gitlab @storage_paths ||= Gitlab.config.repositories.storages end - def with_timeout(args) - %w{timeout 1}.concat(args) + def exec_with_timeout(cmd_args, *args, &block) + Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) end def tmp_file_path(storage_name) @@ -78,7 +79,7 @@ module Gitlab def storage_stat_test(storage_name) stat_path = File.join(path(storage_name), '.') begin - _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} })) + _, status = exec_with_timeout(%W{ stat #{stat_path} }) status == 0 rescue Errno::ENOENT File.exist?(stat_path) && File::Stat.new(stat_path).readable? @@ -86,7 +87,7 @@ module Gitlab end def storage_write_test(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin| + _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| stdin.write(RANDOM_STRING) end status == 0 @@ -96,7 +97,7 @@ module Gitlab end def storage_read_test(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin| + _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| stdin.write(RANDOM_STRING) end status == 0 @@ -106,7 +107,7 @@ module Gitlab end def delete_test_file(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} })) + _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) status == 0 rescue Errno::ENOENT File.delete(tmp_path) rescue Errno::ENOENT diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3411516319f..5ab3eeb3aff 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -12,15 +12,36 @@ module Gitlab AVAILABLE_LANGUAGES.keys end - def set_locale(current_user) - requested_locale = current_user&.preferred_language || ::I18n.default_locale - locale = FastGettext.set_locale(requested_locale) - ::I18n.locale = locale + def locale + FastGettext.locale end - def reset_locale + def locale=(locale_string) + requested_locale = locale_string || ::I18n.default_locale + new_locale = FastGettext.set_locale(requested_locale) + ::I18n.locale = new_locale + end + + def use_default_locale FastGettext.set_locale(::I18n.default_locale) ::I18n.locale = ::I18n.default_locale end + + def with_locale(locale_string) + original_locale = locale + + self.locale = locale_string + yield + ensure + self.locale = original_locale + end + + def with_user_locale(user, &block) + with_locale(user&.preferred_language, &block) + end + + def with_default_locale(&block) + with_locale(::I18n.default_locale, &block) + end end end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 9ad7a38d505..ac9d66c836d 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -22,7 +22,11 @@ module Gitlab def self.config_for(name) name = name.to_s if ldap_provider?(name) - Gitlab::LDAP::Config.new(name).options + if Gitlab::LDAP::Config.valid_provider?(name) + Gitlab::LDAP::Config.new(name).options + else + nil + end else Gitlab.config.omniauth.providers.find { |provider| provider.name == name } end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb new file mode 100644 index 00000000000..9ff6829cd49 --- /dev/null +++ b/lib/gitlab/path_regex.rb @@ -0,0 +1,265 @@ +module Gitlab + module PathRegex + extend self + + # All routes that appear on the top level must be listed here. + # This will make sure that groups cannot be created with these names + # as these routes would be masked by the paths already in place. + # + # Example: + # /api/api-project + # + # the path `api` shouldn't be allowed because it would be masked by `api/*` + # + TOP_LEVEL_ROUTES = %w[ + - + .well-known + abuse_reports + admin + all + api + assets + autocomplete + ci + dashboard + explore + files + groups + health_check + help + hooks + import + invites + issues + jwt + koding + member + merge_requests + new + notes + notification_settings + oauth + profile + projects + public + repository + robots.txt + s + search + sent_notifications + services + snippets + teams + u + unicorn_test + unsubscribes + uploads + users + ].freeze + + # This list should contain all words following `/*namespace_id/:project_id` in + # routes that contain a second wildcard. + # + # Example: + # /*namespace_id/:project_id/badges/*ref/build + # + # If `badges` was allowed as a project/group name, we would not be able to access the + # `badges` route for those projects: + # + # Consider a namespace with path `foo/bar` and a project called `badges`. + # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg` + # + # When accessing this path the route would be matched to the `badges` path + # with the following params: + # - namespace_id: `foo` + # - project_id: `bar` + # - ref: `badges/master` + # + # Failing to find the project, this would result in a 404. + # + # By rejecting `badges` the router can _count_ on the fact that `badges` will + # be preceded by the `namespace/project`. + PROJECT_WILDCARD_ROUTES = %w[ + - + badges + blame + blob + builds + commits + create + create_dir + edit + environments/folders + files + find_file + gitlab-lfs/objects + info/lfs/objects + new + preview + raw + refs + tree + update + wikis + ].freeze + + # These are all the paths that follow `/groups/*id/ or `/groups/*group_id` + # We need to reject these because we have a `/groups/*id` page that is the same + # as the `/*id`. + # + # If we would allow a subgroup to be created with the name `activity` then + # this group would not be accessible through `/groups/parent/activity` since + # this would map to the activity-page of its parent. + GROUP_ROUTES = %w[ + activity + analytics + audit_events + avatar + edit + group_members + hooks + issues + labels + ldap + ldap_group_links + merge_requests + milestones + notification_setting + pipeline_quota + projects + subgroups + ].freeze + + ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES + ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze + + # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript + # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. + # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to + # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of + # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation + # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. + PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + + NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze + NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze + PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze + FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/)*#{NAMESPACE_FORMAT_REGEX}}.freeze + + def root_namespace_route_regex + @root_namespace_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + (?!(#{illegal_words})/) + #{NAMESPACE_FORMAT_REGEX} + }x + end + end + + def full_namespace_route_regex + @full_namespace_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + #{root_namespace_route_regex} + (?: + / + (?!#{illegal_words}/) + #{NAMESPACE_FORMAT_REGEX} + )* + }x + end + end + + def project_route_regex + @project_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + (?!(#{illegal_words})/) + #{PROJECT_PATH_FORMAT_REGEX} + }x + end + end + + def project_git_route_regex + @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze + end + + def root_namespace_path_regex + @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z} + end + + def full_namespace_path_regex + @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} + end + + def project_path_regex + @project_path_regex ||= %r{\A#{project_route_regex}/\z} + end + + def full_project_path_regex + @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} + end + + def full_namespace_format_regex + @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze + end + + def namespace_format_regex + @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/.freeze + end + + def namespace_format_message + "can contain only letters, digits, '_', '-' and '.'. " \ + "Cannot start with '-' or end in '.', '.git' or '.atom'." \ + end + + def project_path_format_regex + @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/.freeze + end + + def project_path_format_message + "can contain only letters, digits, '_', '-' and '.'. " \ + "Cannot start with '-', end in '.git' or end in '.atom'" \ + end + + def archive_formats_regex + # |zip|tar| tar.gz | tar.bz2 | + @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze + end + + def git_reference_regex + # Valid git ref regex, see: + # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html + + @git_reference_regex ||= single_line_regexp %r{ + (?! + (?# doesn't begins with) + \/| (?# rule #6) + (?# doesn't contain) + .*(?: + [\/.]\.| (?# rule #1,3) + \/\/| (?# rule #6) + @\{| (?# rule #8) + \\ (?# rule #9) + ) + ) + [^\000-\040\177~^:?*\[]+ (?# rule #4-5) + (?# doesn't end with) + (?<!\.lock) (?# rule #1) + (?<![\/.]) (?# rule #6-7) + }x + end + + private + + def single_line_regexp(regex) + # Turns a multiline extended regexp into a single line one, + # beacuse `rake routes` breaks on multiline regexes. + Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb new file mode 100644 index 00000000000..bb0df1e3dad --- /dev/null +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -0,0 +1,125 @@ +module Gitlab + module ProjectAuthorizations + # Calculating new project authorizations when supporting nested groups. + # + # This class relies on Common Table Expressions to efficiently get all data, + # including data for nested groups. As a result this class can only be used + # on PostgreSQL. + class WithNestedGroups + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_master_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace. + unscoped. + select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]). + from(cte_alias). + joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace. + unscoped. + select([links[:project_id], + least(cte_alias[:access_level], + links[:group_access], + 'access_level')]). + from(cte_alias). + joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id'). + joins('INNER JOIN projects ON projects.id = project_group_links.project_id'). + joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id'). + where('p_ns.share_with_group_lock IS FALSE') + ] + + union = Gitlab::SQL::Union.new(relations) + + ProjectAuthorization. + unscoped. + with. + recursive(cte.to_arel). + select_from_union(union) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups. + select([namespaces[:id], members[:access_level]]). + except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([namespaces[:id], + greatest(members[:access_level], + cte.table[:access_level], 'access_level')]). + joins(join_cte(cte)). + joins(join_members). + except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id]. + eq(namespaces[:id]). + and(members[:source_type].eq('Namespace')). + and(members[:requested_at].eq(nil)). + and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end + end +end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb new file mode 100644 index 00000000000..627e8c5fba2 --- /dev/null +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -0,0 +1,35 @@ +module Gitlab + module ProjectAuthorizations + # Calculating new project authorizations when not supporting nested groups. + class WithoutNestedGroups + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + relations = [ + # Projects the user is a direct member of + user.projects.select_for_project_authorization, + + # Personal projects + user.personal_projects.select_as_master_for_project_authorization, + + # Projects of groups the user is a member of + user.groups_projects.select_for_project_authorization, + + # Projects shared with groups the user is a member of + user.groups.joins(:shared_projects).select_for_project_authorization + ] + + union = Gitlab::SQL::Union.new(relations) + + ProjectAuthorization. + unscoped. + select_from_union(union) + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b7fef5dd068..e4d2a992470 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,39 +2,6 @@ module Gitlab module Regex extend self - # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript - # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. - # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to - # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_JS` serves as a Javascript-compatible version of - # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation - # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. - PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze - NAMESPACE_REGEX_STR_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze - NO_SUFFIX_REGEX_STR = '(?<!\.git|\.atom)'.freeze - NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR_JS})#{NO_SUFFIX_REGEX_STR}".freeze - PROJECT_REGEX_STR = "(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX_STR}".freeze - - # Same as NAMESPACE_REGEX_STR but allows `/` in the path. - # So `group/subgroup` will match this regex but not NAMESPACE_REGEX_STR - FULL_NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR}/)*#{NAMESPACE_REGEX_STR}".freeze - - def namespace_regex - @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze - end - - def full_namespace_regex - @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} - end - - def namespace_route_regex - @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze - end - - def namespace_regex_message - "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.', '.git' or '.atom'." \ - end - def namespace_name_regex @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -52,23 +19,6 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def project_path_regex - @project_path_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze - end - - def project_route_regex - @project_route_regex ||= /#{PROJECT_REGEX_STR}/.freeze - end - - def project_git_route_regex - @project_route_git_regex ||= /#{PATH_REGEX_STR}\.git/.freeze - end - - def project_path_regex_message - "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-', end in '.git' or end in '.atom'" \ - end - def file_name_regex @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze end @@ -77,36 +27,8 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def archive_formats_regex - # |zip|tar| tar.gz | tar.bz2 | - @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze - end - - def git_reference_regex - # Valid git ref regex, see: - # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html - - @git_reference_regex ||= %r{ - (?! - (?# doesn't begins with) - \/| (?# rule #6) - (?# doesn't contain) - .*(?: - [\/.]\.| (?# rule #1,3) - \/\/| (?# rule #6) - @\{| (?# rule #8) - \\ (?# rule #9) - ) - ) - [^\000-\040\177~^:?*\[]+ (?# rule #4-5) - (?# doesn't end with) - (?<!\.lock) (?# rule #1) - (?<![\/.]) (?# rule #6-7) - }x.freeze - end - def container_registry_reference_regex - git_reference_regex + Gitlab::PathRegex.git_reference_regex end ## diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 36791fae60f..877aa6e6a28 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -25,8 +25,8 @@ module Gitlab def parse_entry(entry) raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) - raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') - raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + raise FormatError, 'Route map entry does not have a source key' unless entry.key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.key?('public') source_pattern = entry['source'] public_path = entry['public'] diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb new file mode 100644 index 00000000000..36d1a8a6f64 --- /dev/null +++ b/lib/gitlab/routes/legacy_builds.rb @@ -0,0 +1,36 @@ +module Gitlab + module Routes + class LegacyBuilds + def initialize(map) + @map = map + end + + def draw + @map.instance_eval do + resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do + collection do + resources :artifacts, only: [], controller: 'build_artifacts' do + collection do + get :latest_succeeded, + path: '*ref_name_and_path', + format: false + end + end + end + + member do + get :raw + end + + resource :artifacts, only: [], controller: 'build_artifacts' do + get :download + get :browse, path: 'browse(/*path)', format: false + get :file, path: 'file/*path', format: false + get :raw, path: 'raw/*path', format: false + end + end + end + end + end + end +end diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb new file mode 100644 index 00000000000..5b1b03820a3 --- /dev/null +++ b/lib/gitlab/sql/recursive_cte.rb @@ -0,0 +1,62 @@ +module Gitlab + module SQL + # Class for easily building recursive CTE statements. + # + # Example: + # + # cte = RecursiveCTE.new(:my_cte_name) + # ns = Arel::Table.new(:namespaces) + # + # cte << Namespace. + # where(ns[:parent_id].eq(some_namespace_id)) + # + # cte << Namespace. + # from([ns, cte.table]). + # where(ns[:parent_id].eq(cte.table[:id])) + # + # Namespace.with. + # recursive(cte.to_arel). + # from(cte.alias_to(ns)) + class RecursiveCTE + attr_reader :table + + # name - The name of the CTE as a String or Symbol. + def initialize(name) + @table = Arel::Table.new(name) + @queries = [] + end + + # Adds a query to the body of the CTE. + # + # relation - The relation object to add to the body of the CTE. + def <<(relation) + @queries << relation + end + + # Returns the Arel relation for this CTE. + def to_arel + sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql) + + Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql)) + end + + # Returns an "AS" statement that aliases the CTE name as the given table + # name. This allows one to trick ActiveRecord into thinking it's selecting + # from an actual table, when in reality it's selecting from a CTE. + # + # alias_table - The Arel table to use as the alias. + def alias_to(alias_table) + Arel::Nodes::As.new(table, alias_table) + end + + # Applies the CTE to the given relation, returning a new one that will + # query from it. + def apply_to(relation) + relation.except(:where). + with. + recursive(to_arel). + from(alias_to(relation.model.arel_table)) + end + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 9ce13feb79a..c81dc7e30d0 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -18,12 +18,6 @@ module Gitlab false end - def self.http_credentials_for_user(user) - return {} unless user.respond_to?(:username) - - { user: user.username } - end - def initialize(url, credentials: nil) @url = Addressable::URI.parse(url.strip) @credentials = credentials diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 4c395b4266e..fa182c4deda 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -21,5 +21,13 @@ module Gitlab nil end + + def boolean_to_yes_no(bool) + if bool + 'Yes' + else + 'No' + end + end end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 2e31f4462f9..85da4c8660b 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -83,7 +83,7 @@ module Gitlab end def valid_level?(level) - options.has_value?(level) + options.value?(level) end def level_name(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index fe37e4da94f..7f27317775c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -31,8 +31,7 @@ module Gitlab feature_enabled = case action.to_s when 'git_receive_pack' - # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172 - false + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) when 'info_refs' @@ -130,7 +129,7 @@ module Gitlab 'MaxSessionTime' => terminal[:max_session_time] } } - details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem) details end |